app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_NET_IXGBE
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIB_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIB_BITRATESTATS
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIB_LATENCYSTATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use main core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 200         DEFAULT_MBUF_DATA_SIZE
 201 }; /**< Mbuf data space size. */
 202 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 203                                       * specified on command-line. */
 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 205
 206 /*
 207  * In container, it cannot terminate the process which running with 'stats-period'
 208  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 209  */
 210 uint8_t f_quit;
 211
 212 /*
 213  * Configuration of packet segments used to scatter received packets
 214  * if some of split features is configured.
 215  */
 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 217 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 219 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 220
 221 /*
 222  * Configuration of packet segments used by the "txonly" processing engine.
 223  */
 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 226         TXONLY_DEF_PACKET_LEN,
 227 };
 228 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 229
 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 231 /**< Split policy for packets to TX. */
 232
 233 uint8_t txonly_multi_flow;
 234 /**< Whether multiple flows are generated in TXONLY mode. */
 235
 236 uint32_t tx_pkt_times_inter;
 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 238
 239 uint32_t tx_pkt_times_intra;
 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 241
 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 243 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 244 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 245
 246 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 247 uint8_t dcb_config = 0;
 248
 249 /* Whether the dcb is in testing status */
 250 uint8_t dcb_test = 0;
 251
 252 /*
 253  * Configurable number of RX/TX queues.
 254  */
 255 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 256 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 257 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 258
 259 /*
 260  * Configurable number of RX/TX ring descriptors.
 261  * Defaults are supplied by drivers via ethdev.
 262  */
 263 #define RTE_TEST_RX_DESC_DEFAULT 0
 264 #define RTE_TEST_TX_DESC_DEFAULT 0
 265 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 266 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 267
 268 #define RTE_PMD_PARAM_UNSET -1
 269 /*
 270  * Configurable values of RX and TX ring threshold registers.
 271  */
 272
 273 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 274 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 275 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 276
 277 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 278 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 279 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 280
 281 /*
 282  * Configurable value of RX free threshold.
 283  */
 284 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 285
 286 /*
 287  * Configurable value of RX drop enable.
 288  */
 289 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 290
 291 /*
 292  * Configurable value of TX free threshold.
 293  */
 294 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 295
 296 /*
 297  * Configurable value of TX RS bit threshold.
 298  */
 299 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 300
 301 /*
 302  * Configurable value of buffered packets before sending.
 303  */
 304 uint16_t noisy_tx_sw_bufsz;
 305
 306 /*
 307  * Configurable value of packet buffer timeout.
 308  */
 309 uint16_t noisy_tx_sw_buf_flush_time;
 310
 311 /*
 312  * Configurable value for size of VNF internal memory area
 313  * used for simulating noisy neighbour behaviour
 314  */
 315 uint64_t noisy_lkup_mem_sz;
 316
 317 /*
 318  * Configurable value of number of random writes done in
 319  * VNF simulation memory area.
 320  */
 321 uint64_t noisy_lkup_num_writes;
 322
 323 /*
 324  * Configurable value of number of random reads done in
 325  * VNF simulation memory area.
 326  */
 327 uint64_t noisy_lkup_num_reads;
 328
 329 /*
 330  * Configurable value of number of random reads/writes done in
 331  * VNF simulation memory area.
 332  */
 333 uint64_t noisy_lkup_num_reads_writes;
 334
 335 /*
 336  * Receive Side Scaling (RSS) configuration.
 337  */
 338 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 339
 340 /*
 341  * Port topology configuration
 342  */
 343 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 344
 345 /*
 346  * Avoids to flush all the RX streams before starts forwarding.
 347  */
 348 uint8_t no_flush_rx = 0; /* flush by default */
 349
 350 /*
 351  * Flow API isolated mode.
 352  */
 353 uint8_t flow_isolate_all;
 354
 355 /*
 356  * Avoids to check link status when starting/stopping a port.
 357  */
 358 uint8_t no_link_check = 0; /* check by default */
 359
 360 /*
 361  * Don't automatically start all ports in interactive mode.
 362  */
 363 uint8_t no_device_start = 0;
 364
 365 /*
 366  * Enable link status change notification
 367  */
 368 uint8_t lsc_interrupt = 1; /* enabled by default */
 369
 370 /*
 371  * Enable device removal notification.
 372  */
 373 uint8_t rmv_interrupt = 1; /* enabled by default */
 374
 375 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 376
 377 /* After attach, port setup is called on event or by iterator */
 378 bool setup_on_probe_event = true;
 379
 380 /* Clear ptypes on port initialization. */
 381 uint8_t clear_ptypes = true;
 382
 383 /* Hairpin ports configuration mode. */
 384 uint16_t hairpin_mode;
 385
 386 /* Pretty printing of ethdev events */
 387 static const char * const eth_event_desc[] = {
 388         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 389         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 390         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 391         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 392         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 393         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 394         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 395         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 396         [RTE_ETH_EVENT_NEW] = "device probed",
 397         [RTE_ETH_EVENT_DESTROY] = "device released",
 398         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 399         [RTE_ETH_EVENT_MAX] = NULL,
 400 };
 401
 402 /*
 403  * Display or mask ether events
 404  * Default to all events except VF_MBOX
 405  */
 406 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 407                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 408                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 413                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 414 /*
 415  * Decide if all memory are locked for performance.
 416  */
 417 int do_mlockall = 0;
 418
 419 /*
 420  * NIC bypass mode configuration options.
 421  */
 422
 423 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 424 /* The NIC bypass watchdog timeout. */
 425 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 426 #endif
 427
 428
 429 #ifdef RTE_LIB_LATENCYSTATS
 430
 431 /*
 432  * Set when latency stats is enabled in the commandline
 433  */
 434 uint8_t latencystats_enabled;
 435
 436 /*
 437  * Lcore ID to serive latency statistics.
 438  */
 439 lcoreid_t latencystats_lcore_id = -1;
 440
 441 #endif
 442
 443 /*
 444  * Ethernet device configuration.
 445  */
 446 struct rte_eth_rxmode rx_mode = {
 447         /* Default maximum frame length.
 448          * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
 449          * in init_config().
 450          */
 451         .max_rx_pkt_len = 0,
 452 };
 453
 454 struct rte_eth_txmode tx_mode = {
 455         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 456 };
 457
 458 struct rte_fdir_conf fdir_conf = {
 459         .mode = RTE_FDIR_MODE_NONE,
 460         .pballoc = RTE_FDIR_PBALLOC_64K,
 461         .status = RTE_FDIR_REPORT_STATUS,
 462         .mask = {
 463                 .vlan_tci_mask = 0xFFEF,
 464                 .ipv4_mask     = {
 465                         .src_ip = 0xFFFFFFFF,
 466                         .dst_ip = 0xFFFFFFFF,
 467                 },
 468                 .ipv6_mask     = {
 469                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 470                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 471                 },
 472                 .src_port_mask = 0xFFFF,
 473                 .dst_port_mask = 0xFFFF,
 474                 .mac_addr_byte_mask = 0xFF,
 475                 .tunnel_type_mask = 1,
 476                 .tunnel_id_mask = 0xFFFFFFFF,
 477         },
 478         .drop_queue = 127,
 479 };
 480
 481 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 482
 483 /*
 484  * Display zero values by default for xstats
 485  */
 486 uint8_t xstats_hide_zero;
 487
 488 /*
 489  * Measure of CPU cycles disabled by default
 490  */
 491 uint8_t record_core_cycles;
 492
 493 /*
 494  * Display of RX and TX bursts disabled by default
 495  */
 496 uint8_t record_burst_stats;
 497
 498 unsigned int num_sockets = 0;
 499 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 500
 501 #ifdef RTE_LIB_BITRATESTATS
 502 /* Bitrate statistics */
 503 struct rte_stats_bitrates *bitrate_data;
 504 lcoreid_t bitrate_lcore_id;
 505 uint8_t bitrate_enabled;
 506 #endif
 507
 508 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 509 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 510
 511 /*
 512  * hexadecimal bitmask of RX mq mode can be enabled.
 513  */
 514 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 515
 516 /* Forward function declarations */
 517 static void setup_attached_port(portid_t pi);
 518 static void check_all_ports_link_status(uint32_t port_mask);
 519 static int eth_event_callback(portid_t port_id,
 520                               enum rte_eth_event_type type,
 521                               void *param, void *ret_param);
 522 static void dev_event_callback(const char *device_name,
 523                                 enum rte_dev_event_type type,
 524                                 void *param);
 525
 526 /*
 527  * Check if all the ports are started.
 528  * If yes, return positive value. If not, return zero.
 529  */
 530 static int all_ports_started(void);
 531
 532 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 533 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 534
 535 /* Holds the registered mbuf dynamic flags names. */
 536 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 537
 538 /*
 539  * Helper function to check if socket is already discovered.
 540  * If yes, return positive value. If not, return zero.
 541  */
 542 int
 543 new_socket_id(unsigned int socket_id)
 544 {
 545         unsigned int i;
 546
 547         for (i = 0; i < num_sockets; i++) {
 548                 if (socket_ids[i] == socket_id)
 549                         return 0;
 550         }
 551         return 1;
 552 }
 553
 554 /*
 555  * Setup default configuration.
 556  */
 557 static void
 558 set_default_fwd_lcores_config(void)
 559 {
 560         unsigned int i;
 561         unsigned int nb_lc;
 562         unsigned int sock_num;
 563
 564         nb_lc = 0;
 565         for (i = 0; i < RTE_MAX_LCORE; i++) {
 566                 if (!rte_lcore_is_enabled(i))
 567                         continue;
 568                 sock_num = rte_lcore_to_socket_id(i);
 569                 if (new_socket_id(sock_num)) {
 570                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 571                                 rte_exit(EXIT_FAILURE,
 572                                          "Total sockets greater than %u\n",
 573                                          RTE_MAX_NUMA_NODES);
 574                         }
 575                         socket_ids[num_sockets++] = sock_num;
 576                 }
 577                 if (i == rte_get_main_lcore())
 578                         continue;
 579                 fwd_lcores_cpuids[nb_lc++] = i;
 580         }
 581         nb_lcores = (lcoreid_t) nb_lc;
 582         nb_cfg_lcores = nb_lcores;
 583         nb_fwd_lcores = 1;
 584 }
 585
 586 static void
 587 set_def_peer_eth_addrs(void)
 588 {
 589         portid_t i;
 590
 591         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 592                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 593                 peer_eth_addrs[i].addr_bytes[5] = i;
 594         }
 595 }
 596
 597 static void
 598 set_default_fwd_ports_config(void)
 599 {
 600         portid_t pt_id;
 601         int i = 0;
 602
 603         RTE_ETH_FOREACH_DEV(pt_id) {
 604                 fwd_ports_ids[i++] = pt_id;
 605
 606                 /* Update sockets info according to the attached device */
 607                 int socket_id = rte_eth_dev_socket_id(pt_id);
 608                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 609                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 610                                 rte_exit(EXIT_FAILURE,
 611                                          "Total sockets greater than %u\n",
 612                                          RTE_MAX_NUMA_NODES);
 613                         }
 614                         socket_ids[num_sockets++] = socket_id;
 615                 }
 616         }
 617
 618         nb_cfg_ports = nb_ports;
 619         nb_fwd_ports = nb_ports;
 620 }
 621
 622 void
 623 set_def_fwd_config(void)
 624 {
 625         set_default_fwd_lcores_config();
 626         set_def_peer_eth_addrs();
 627         set_default_fwd_ports_config();
 628 }
 629
 630 /* extremely pessimistic estimation of memory required to create a mempool */
 631 static int
 632 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 633 {
 634         unsigned int n_pages, mbuf_per_pg, leftover;
 635         uint64_t total_mem, mbuf_mem, obj_sz;
 636
 637         /* there is no good way to predict how much space the mempool will
 638          * occupy because it will allocate chunks on the fly, and some of those
 639          * will come from default DPDK memory while some will come from our
 640          * external memory, so just assume 128MB will be enough for everyone.
 641          */
 642         uint64_t hdr_mem = 128 << 20;
 643
 644         /* account for possible non-contiguousness */
 645         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 646         if (obj_sz > pgsz) {
 647                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 648                 return -1;
 649         }
 650
 651         mbuf_per_pg = pgsz / obj_sz;
 652         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 653         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 654
 655         mbuf_mem = n_pages * pgsz;
 656
 657         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 658
 659         if (total_mem > SIZE_MAX) {
 660                 TESTPMD_LOG(ERR, "Memory size too big\n");
 661                 return -1;
 662         }
 663         *out = (size_t)total_mem;
 664
 665         return 0;
 666 }
 667
 668 static int
 669 pagesz_flags(uint64_t page_sz)
 670 {
 671         /* as per mmap() manpage, all page sizes are log2 of page size
 672          * shifted by MAP_HUGE_SHIFT
 673          */
 674         int log2 = rte_log2_u64(page_sz);
 675
 676         return (log2 << HUGE_SHIFT);
 677 }
 678
 679 static void *
 680 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 681 {
 682         void *addr;
 683         int flags;
 684
 685         /* allocate anonymous hugepages */
 686         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 687         if (huge)
 688                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 689
 690         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 691         if (addr == MAP_FAILED)
 692                 return NULL;
 693
 694         return addr;
 695 }
 696
 697 struct extmem_param {
 698         void *addr;
 699         size_t len;
 700         size_t pgsz;
 701         rte_iova_t *iova_table;
 702         unsigned int iova_table_len;
 703 };
 704
 705 static int
 706 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 707                 bool huge)
 708 {
 709         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 710                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 711         unsigned int cur_page, n_pages, pgsz_idx;
 712         size_t mem_sz, cur_pgsz;
 713         rte_iova_t *iovas = NULL;
 714         void *addr;
 715         int ret;
 716
 717         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 718                 /* skip anything that is too big */
 719                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 720                         continue;
 721
 722                 cur_pgsz = pgsizes[pgsz_idx];
 723
 724                 /* if we were told not to allocate hugepages, override */
 725                 if (!huge)
 726                         cur_pgsz = sysconf(_SC_PAGESIZE);
 727
 728                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 729                 if (ret < 0) {
 730                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 731                         return -1;
 732                 }
 733
 734                 /* allocate our memory */
 735                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 736
 737                 /* if we couldn't allocate memory with a specified page size,
 738                  * that doesn't mean we can't do it with other page sizes, so
 739                  * try another one.
 740                  */
 741                 if (addr == NULL)
 742                         continue;
 743
 744                 /* store IOVA addresses for every page in this memory area */
 745                 n_pages = mem_sz / cur_pgsz;
 746
 747                 iovas = malloc(sizeof(*iovas) * n_pages);
 748
 749                 if (iovas == NULL) {
 750                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 751                         goto fail;
 752                 }
 753                 /* lock memory if it's not huge pages */
 754                 if (!huge)
 755                         mlock(addr, mem_sz);
 756
 757                 /* populate IOVA addresses */
 758                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 759                         rte_iova_t iova;
 760                         size_t offset;
 761                         void *cur;
 762
 763                         offset = cur_pgsz * cur_page;
 764                         cur = RTE_PTR_ADD(addr, offset);
 765
 766                         /* touch the page before getting its IOVA */
 767                         *(volatile char *)cur = 0;
 768
 769                         iova = rte_mem_virt2iova(cur);
 770
 771                         iovas[cur_page] = iova;
 772                 }
 773
 774                 break;
 775         }
 776         /* if we couldn't allocate anything */
 777         if (iovas == NULL)
 778                 return -1;
 779
 780         param->addr = addr;
 781         param->len = mem_sz;
 782         param->pgsz = cur_pgsz;
 783         param->iova_table = iovas;
 784         param->iova_table_len = n_pages;
 785
 786         return 0;
 787 fail:
 788         if (iovas)
 789                 free(iovas);
 790         if (addr)
 791                 munmap(addr, mem_sz);
 792
 793         return -1;
 794 }
 795
 796 static int
 797 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 798 {
 799         struct extmem_param param;
 800         int socket_id, ret;
 801
 802         memset(&param, 0, sizeof(param));
 803
 804         /* check if our heap exists */
 805         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 806         if (socket_id < 0) {
 807                 /* create our heap */
 808                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 809                 if (ret < 0) {
 810                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 811                         return -1;
 812                 }
 813         }
 814
 815         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 816         if (ret < 0) {
 817                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 818                 return -1;
 819         }
 820
 821         /* we now have a valid memory area, so add it to heap */
 822         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 823                         param.addr, param.len, param.iova_table,
 824                         param.iova_table_len, param.pgsz);
 825
 826         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 827
 828         /* not needed any more */
 829         free(param.iova_table);
 830
 831         if (ret < 0) {
 832                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 833                 munmap(param.addr, param.len);
 834                 return -1;
 835         }
 836
 837         /* success */
 838
 839         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 840                         param.len >> 20);
 841
 842         return 0;
 843 }
 844 static void
 845 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 846              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 847 {
 848         uint16_t pid = 0;
 849         int ret;
 850
 851         RTE_ETH_FOREACH_DEV(pid) {
 852                 struct rte_eth_dev *dev =
 853                         &rte_eth_devices[pid];
 854
 855                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 856                                         memhdr->len);
 857                 if (ret) {
 858                         TESTPMD_LOG(DEBUG,
 859                                     "unable to DMA unmap addr 0x%p "
 860                                     "for device %s\n",
 861                                     memhdr->addr, dev->data->name);
 862                 }
 863         }
 864         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 865         if (ret) {
 866                 TESTPMD_LOG(DEBUG,
 867                             "unable to un-register addr 0x%p\n", memhdr->addr);
 868         }
 869 }
 870
 871 static void
 872 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 873            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 874 {
 875         uint16_t pid = 0;
 876         size_t page_size = sysconf(_SC_PAGESIZE);
 877         int ret;
 878
 879         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 880                                   page_size);
 881         if (ret) {
 882                 TESTPMD_LOG(DEBUG,
 883                             "unable to register addr 0x%p\n", memhdr->addr);
 884                 return;
 885         }
 886         RTE_ETH_FOREACH_DEV(pid) {
 887                 struct rte_eth_dev *dev =
 888                         &rte_eth_devices[pid];
 889
 890                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 891                                       memhdr->len);
 892                 if (ret) {
 893                         TESTPMD_LOG(DEBUG,
 894                                     "unable to DMA map addr 0x%p "
 895                                     "for device %s\n",
 896                                     memhdr->addr, dev->data->name);
 897                 }
 898         }
 899 }
 900
 901 static unsigned int
 902 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 903             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 904 {
 905         struct rte_pktmbuf_extmem *xmem;
 906         unsigned int ext_num, zone_num, elt_num;
 907         uint16_t elt_size;
 908
 909         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 910         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 911         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 912
 913         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 914         if (xmem == NULL) {
 915                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 916                                  "external buffer descriptors\n");
 917                 *ext_mem = NULL;
 918                 return 0;
 919         }
 920         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 921                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 922                 const struct rte_memzone *mz;
 923                 char mz_name[RTE_MEMZONE_NAMESIZE];
 924                 int ret;
 925
 926                 ret = snprintf(mz_name, sizeof(mz_name),
 927                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 928                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 929                         errno = ENAMETOOLONG;
 930                         ext_num = 0;
 931                         break;
 932                 }
 933                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 934                                                  socket_id,
 935                                                  RTE_MEMZONE_IOVA_CONTIG |
 936                                                  RTE_MEMZONE_1GB |
 937                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 938                                                  EXTBUF_ZONE_SIZE);
 939                 if (mz == NULL) {
 940                         /*
 941                          * The caller exits on external buffer creation
 942                          * error, so there is no need to free memzones.
 943                          */
 944                         errno = ENOMEM;
 945                         ext_num = 0;
 946                         break;
 947                 }
 948                 xseg->buf_ptr = mz->addr;
 949                 xseg->buf_iova = mz->iova;
 950                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 951                 xseg->elt_size = elt_size;
 952         }
 953         if (ext_num == 0 && xmem != NULL) {
 954                 free(xmem);
 955                 xmem = NULL;
 956         }
 957         *ext_mem = xmem;
 958         return ext_num;
 959 }
 960
 961 /*
 962  * Configuration initialisation done once at init time.
 963  */
 964 static struct rte_mempool *
 965 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 966                  unsigned int socket_id, uint16_t size_idx)
 967 {
 968         char pool_name[RTE_MEMPOOL_NAMESIZE];
 969         struct rte_mempool *rte_mp = NULL;
 970         uint32_t mb_size;
 971
 972         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 973         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 974
 975         TESTPMD_LOG(INFO,
 976                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 977                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 978
 979         switch (mp_alloc_type) {
 980         case MP_ALLOC_NATIVE:
 981                 {
 982                         /* wrapper to rte_mempool_create() */
 983                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 984                                         rte_mbuf_best_mempool_ops());
 985                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 986                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 987                         break;
 988                 }
 989         case MP_ALLOC_ANON:
 990                 {
 991                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 992                                 mb_size, (unsigned int) mb_mempool_cache,
 993                                 sizeof(struct rte_pktmbuf_pool_private),
 994                                 socket_id, mempool_flags);
 995                         if (rte_mp == NULL)
 996                                 goto err;
 997
 998                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 999                                 rte_mempool_free(rte_mp);
1000                                 rte_mp = NULL;
1001                                 goto err;
1002                         }
1003                         rte_pktmbuf_pool_init(rte_mp, NULL);
1004                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1005                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1006                         break;
1007                 }
1008         case MP_ALLOC_XMEM:
1009         case MP_ALLOC_XMEM_HUGE:
1010                 {
1011                         int heap_socket;
1012                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1013
1014                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1015                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1016
1017                         heap_socket =
1018                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1019                         if (heap_socket < 0)
1020                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1021
1022                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1023                                         rte_mbuf_best_mempool_ops());
1024                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1025                                         mb_mempool_cache, 0, mbuf_seg_size,
1026                                         heap_socket);
1027                         break;
1028                 }
1029         case MP_ALLOC_XBUF:
1030                 {
1031                         struct rte_pktmbuf_extmem *ext_mem;
1032                         unsigned int ext_num;
1033
1034                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1035                                                socket_id, pool_name, &ext_mem);
1036                         if (ext_num == 0)
1037                                 rte_exit(EXIT_FAILURE,
1038                                          "Can't create pinned data buffers\n");
1039
1040                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1041                                         rte_mbuf_best_mempool_ops());
1042                         rte_mp = rte_pktmbuf_pool_create_extbuf
1043                                         (pool_name, nb_mbuf, mb_mempool_cache,
1044                                          0, mbuf_seg_size, socket_id,
1045                                          ext_mem, ext_num);
1046                         free(ext_mem);
1047                         break;
1048                 }
1049         default:
1050                 {
1051                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1052                 }
1053         }
1054
1055 err:
1056         if (rte_mp == NULL) {
1057                 rte_exit(EXIT_FAILURE,
1058                         "Creation of mbuf pool for socket %u failed: %s\n",
1059                         socket_id, rte_strerror(rte_errno));
1060         } else if (verbose_level > 0) {
1061                 rte_mempool_dump(stdout, rte_mp);
1062         }
1063         return rte_mp;
1064 }
1065
1066 /*
1067  * Check given socket id is valid or not with NUMA mode,
1068  * if valid, return 0, else return -1
1069  */
1070 static int
1071 check_socket_id(const unsigned int socket_id)
1072 {
1073         static int warning_once = 0;
1074
1075         if (new_socket_id(socket_id)) {
1076                 if (!warning_once && numa_support)
1077                         printf("Warning: NUMA should be configured manually by"
1078                                " using --port-numa-config and"
1079                                " --ring-numa-config parameters along with"
1080                                " --numa.\n");
1081                 warning_once = 1;
1082                 return -1;
1083         }
1084         return 0;
1085 }
1086
1087 /*
1088  * Get the allowed maximum number of RX queues.
1089  * *pid return the port id which has minimal value of
1090  * max_rx_queues in all ports.
1091  */
1092 queueid_t
1093 get_allowed_max_nb_rxq(portid_t *pid)
1094 {
1095         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1096         bool max_rxq_valid = false;
1097         portid_t pi;
1098         struct rte_eth_dev_info dev_info;
1099
1100         RTE_ETH_FOREACH_DEV(pi) {
1101                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1102                         continue;
1103
1104                 max_rxq_valid = true;
1105                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1106                         allowed_max_rxq = dev_info.max_rx_queues;
1107                         *pid = pi;
1108                 }
1109         }
1110         return max_rxq_valid ? allowed_max_rxq : 0;
1111 }
1112
1113 /*
1114  * Check input rxq is valid or not.
1115  * If input rxq is not greater than any of maximum number
1116  * of RX queues of all ports, it is valid.
1117  * if valid, return 0, else return -1
1118  */
1119 int
1120 check_nb_rxq(queueid_t rxq)
1121 {
1122         queueid_t allowed_max_rxq;
1123         portid_t pid = 0;
1124
1125         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1126         if (rxq > allowed_max_rxq) {
1127                 printf("Fail: input rxq (%u) can't be greater "
1128                        "than max_rx_queues (%u) of port %u\n",
1129                        rxq,
1130                        allowed_max_rxq,
1131                        pid);
1132                 return -1;
1133         }
1134         return 0;
1135 }
1136
1137 /*
1138  * Get the allowed maximum number of TX queues.
1139  * *pid return the port id which has minimal value of
1140  * max_tx_queues in all ports.
1141  */
1142 queueid_t
1143 get_allowed_max_nb_txq(portid_t *pid)
1144 {
1145         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1146         bool max_txq_valid = false;
1147         portid_t pi;
1148         struct rte_eth_dev_info dev_info;
1149
1150         RTE_ETH_FOREACH_DEV(pi) {
1151                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1152                         continue;
1153
1154                 max_txq_valid = true;
1155                 if (dev_info.max_tx_queues < allowed_max_txq) {
1156                         allowed_max_txq = dev_info.max_tx_queues;
1157                         *pid = pi;
1158                 }
1159         }
1160         return max_txq_valid ? allowed_max_txq : 0;
1161 }
1162
1163 /*
1164  * Check input txq is valid or not.
1165  * If input txq is not greater than any of maximum number
1166  * of TX queues of all ports, it is valid.
1167  * if valid, return 0, else return -1
1168  */
1169 int
1170 check_nb_txq(queueid_t txq)
1171 {
1172         queueid_t allowed_max_txq;
1173         portid_t pid = 0;
1174
1175         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1176         if (txq > allowed_max_txq) {
1177                 printf("Fail: input txq (%u) can't be greater "
1178                        "than max_tx_queues (%u) of port %u\n",
1179                        txq,
1180                        allowed_max_txq,
1181                        pid);
1182                 return -1;
1183         }
1184         return 0;
1185 }
1186
1187 /*
1188  * Get the allowed maximum number of RXDs of every rx queue.
1189  * *pid return the port id which has minimal value of
1190  * max_rxd in all queues of all ports.
1191  */
1192 static uint16_t
1193 get_allowed_max_nb_rxd(portid_t *pid)
1194 {
1195         uint16_t allowed_max_rxd = UINT16_MAX;
1196         portid_t pi;
1197         struct rte_eth_dev_info dev_info;
1198
1199         RTE_ETH_FOREACH_DEV(pi) {
1200                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1201                         continue;
1202
1203                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1204                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1205                         *pid = pi;
1206                 }
1207         }
1208         return allowed_max_rxd;
1209 }
1210
1211 /*
1212  * Get the allowed minimal number of RXDs of every rx queue.
1213  * *pid return the port id which has minimal value of
1214  * min_rxd in all queues of all ports.
1215  */
1216 static uint16_t
1217 get_allowed_min_nb_rxd(portid_t *pid)
1218 {
1219         uint16_t allowed_min_rxd = 0;
1220         portid_t pi;
1221         struct rte_eth_dev_info dev_info;
1222
1223         RTE_ETH_FOREACH_DEV(pi) {
1224                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1225                         continue;
1226
1227                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1228                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1229                         *pid = pi;
1230                 }
1231         }
1232
1233         return allowed_min_rxd;
1234 }
1235
1236 /*
1237  * Check input rxd is valid or not.
1238  * If input rxd is not greater than any of maximum number
1239  * of RXDs of every Rx queues and is not less than any of
1240  * minimal number of RXDs of every Rx queues, it is valid.
1241  * if valid, return 0, else return -1
1242  */
1243 int
1244 check_nb_rxd(queueid_t rxd)
1245 {
1246         uint16_t allowed_max_rxd;
1247         uint16_t allowed_min_rxd;
1248         portid_t pid = 0;
1249
1250         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1251         if (rxd > allowed_max_rxd) {
1252                 printf("Fail: input rxd (%u) can't be greater "
1253                        "than max_rxds (%u) of port %u\n",
1254                        rxd,
1255                        allowed_max_rxd,
1256                        pid);
1257                 return -1;
1258         }
1259
1260         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1261         if (rxd < allowed_min_rxd) {
1262                 printf("Fail: input rxd (%u) can't be less "
1263                        "than min_rxds (%u) of port %u\n",
1264                        rxd,
1265                        allowed_min_rxd,
1266                        pid);
1267                 return -1;
1268         }
1269
1270         return 0;
1271 }
1272
1273 /*
1274  * Get the allowed maximum number of TXDs of every rx queues.
1275  * *pid return the port id which has minimal value of
1276  * max_txd in every tx queue.
1277  */
1278 static uint16_t
1279 get_allowed_max_nb_txd(portid_t *pid)
1280 {
1281         uint16_t allowed_max_txd = UINT16_MAX;
1282         portid_t pi;
1283         struct rte_eth_dev_info dev_info;
1284
1285         RTE_ETH_FOREACH_DEV(pi) {
1286                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1287                         continue;
1288
1289                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1290                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1291                         *pid = pi;
1292                 }
1293         }
1294         return allowed_max_txd;
1295 }
1296
1297 /*
1298  * Get the allowed maximum number of TXDs of every tx queues.
1299  * *pid return the port id which has minimal value of
1300  * min_txd in every tx queue.
1301  */
1302 static uint16_t
1303 get_allowed_min_nb_txd(portid_t *pid)
1304 {
1305         uint16_t allowed_min_txd = 0;
1306         portid_t pi;
1307         struct rte_eth_dev_info dev_info;
1308
1309         RTE_ETH_FOREACH_DEV(pi) {
1310                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1311                         continue;
1312
1313                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1314                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1315                         *pid = pi;
1316                 }
1317         }
1318
1319         return allowed_min_txd;
1320 }
1321
1322 /*
1323  * Check input txd is valid or not.
1324  * If input txd is not greater than any of maximum number
1325  * of TXDs of every Rx queues, it is valid.
1326  * if valid, return 0, else return -1
1327  */
1328 int
1329 check_nb_txd(queueid_t txd)
1330 {
1331         uint16_t allowed_max_txd;
1332         uint16_t allowed_min_txd;
1333         portid_t pid = 0;
1334
1335         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1336         if (txd > allowed_max_txd) {
1337                 printf("Fail: input txd (%u) can't be greater "
1338                        "than max_txds (%u) of port %u\n",
1339                        txd,
1340                        allowed_max_txd,
1341                        pid);
1342                 return -1;
1343         }
1344
1345         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1346         if (txd < allowed_min_txd) {
1347                 printf("Fail: input txd (%u) can't be less "
1348                        "than min_txds (%u) of port %u\n",
1349                        txd,
1350                        allowed_min_txd,
1351                        pid);
1352                 return -1;
1353         }
1354         return 0;
1355 }
1356
1357
1358 /*
1359  * Get the allowed maximum number of hairpin queues.
1360  * *pid return the port id which has minimal value of
1361  * max_hairpin_queues in all ports.
1362  */
1363 queueid_t
1364 get_allowed_max_nb_hairpinq(portid_t *pid)
1365 {
1366         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1367         portid_t pi;
1368         struct rte_eth_hairpin_cap cap;
1369
1370         RTE_ETH_FOREACH_DEV(pi) {
1371                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1372                         *pid = pi;
1373                         return 0;
1374                 }
1375                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1376                         allowed_max_hairpinq = cap.max_nb_queues;
1377                         *pid = pi;
1378                 }
1379         }
1380         return allowed_max_hairpinq;
1381 }
1382
1383 /*
1384  * Check input hairpin is valid or not.
1385  * If input hairpin is not greater than any of maximum number
1386  * of hairpin queues of all ports, it is valid.
1387  * if valid, return 0, else return -1
1388  */
1389 int
1390 check_nb_hairpinq(queueid_t hairpinq)
1391 {
1392         queueid_t allowed_max_hairpinq;
1393         portid_t pid = 0;
1394
1395         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1396         if (hairpinq > allowed_max_hairpinq) {
1397                 printf("Fail: input hairpin (%u) can't be greater "
1398                        "than max_hairpin_queues (%u) of port %u\n",
1399                        hairpinq, allowed_max_hairpinq, pid);
1400                 return -1;
1401         }
1402         return 0;
1403 }
1404
1405 static void
1406 init_config(void)
1407 {
1408         portid_t pid;
1409         struct rte_port *port;
1410         struct rte_mempool *mbp;
1411         unsigned int nb_mbuf_per_pool;
1412         lcoreid_t  lc_id;
1413         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1414         struct rte_gro_param gro_param;
1415         uint32_t gso_types;
1416         uint16_t data_size;
1417         bool warning = 0;
1418         int k;
1419         int ret;
1420
1421         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1422
1423         /* Configuration of logical cores. */
1424         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1425                                 sizeof(struct fwd_lcore *) * nb_lcores,
1426                                 RTE_CACHE_LINE_SIZE);
1427         if (fwd_lcores == NULL) {
1428                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1429                                                         "failed\n", nb_lcores);
1430         }
1431         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1432                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1433                                                sizeof(struct fwd_lcore),
1434                                                RTE_CACHE_LINE_SIZE);
1435                 if (fwd_lcores[lc_id] == NULL) {
1436                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1437                                                                 "failed\n");
1438                 }
1439                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1440         }
1441
1442         RTE_ETH_FOREACH_DEV(pid) {
1443                 port = &ports[pid];
1444                 /* Apply default TxRx configuration for all ports */
1445                 port->dev_conf.txmode = tx_mode;
1446                 port->dev_conf.rxmode = rx_mode;
1447
1448                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1449                 if (ret != 0)
1450                         rte_exit(EXIT_FAILURE,
1451                                  "rte_eth_dev_info_get() failed\n");
1452
1453                 ret = update_jumbo_frame_offload(pid);
1454                 if (ret != 0)
1455                         printf("Updating jumbo frame offload failed for port %u\n",
1456                                 pid);
1457
1458                 if (!(port->dev_info.tx_offload_capa &
1459                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1460                         port->dev_conf.txmode.offloads &=
1461                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1462                 if (numa_support) {
1463                         if (port_numa[pid] != NUMA_NO_CONFIG)
1464                                 port_per_socket[port_numa[pid]]++;
1465                         else {
1466                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1467
1468                                 /*
1469                                  * if socket_id is invalid,
1470                                  * set to the first available socket.
1471                                  */
1472                                 if (check_socket_id(socket_id) < 0)
1473                                         socket_id = socket_ids[0];
1474                                 port_per_socket[socket_id]++;
1475                         }
1476                 }
1477
1478                 /* Apply Rx offloads configuration */
1479                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1480                         port->rx_conf[k].offloads =
1481                                 port->dev_conf.rxmode.offloads;
1482                 /* Apply Tx offloads configuration */
1483                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1484                         port->tx_conf[k].offloads =
1485                                 port->dev_conf.txmode.offloads;
1486
1487                 /* set flag to initialize port/queue */
1488                 port->need_reconfig = 1;
1489                 port->need_reconfig_queues = 1;
1490                 port->tx_metadata = 0;
1491
1492                 /* Check for maximum number of segments per MTU. Accordingly
1493                  * update the mbuf data size.
1494                  */
1495                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1496                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1497                         data_size = rx_mode.max_rx_pkt_len /
1498                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1499
1500                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1501                                                         mbuf_data_size[0]) {
1502                                 mbuf_data_size[0] = data_size +
1503                                                  RTE_PKTMBUF_HEADROOM;
1504                                 warning = 1;
1505                         }
1506                 }
1507         }
1508
1509         if (warning)
1510                 TESTPMD_LOG(WARNING,
1511                             "Configured mbuf size of the first segment %hu\n",
1512                             mbuf_data_size[0]);
1513         /*
1514          * Create pools of mbuf.
1515          * If NUMA support is disabled, create a single pool of mbuf in
1516          * socket 0 memory by default.
1517          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1518          *
1519          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1520          * nb_txd can be configured at run time.
1521          */
1522         if (param_total_num_mbufs)
1523                 nb_mbuf_per_pool = param_total_num_mbufs;
1524         else {
1525                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1526                         (nb_lcores * mb_mempool_cache) +
1527                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1528                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1529         }
1530
1531         if (numa_support) {
1532                 uint8_t i, j;
1533
1534                 for (i = 0; i < num_sockets; i++)
1535                         for (j = 0; j < mbuf_data_size_n; j++)
1536                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1537                                         mbuf_pool_create(mbuf_data_size[j],
1538                                                           nb_mbuf_per_pool,
1539                                                           socket_ids[i], j);
1540         } else {
1541                 uint8_t i;
1542
1543                 for (i = 0; i < mbuf_data_size_n; i++)
1544                         mempools[i] = mbuf_pool_create
1545                                         (mbuf_data_size[i],
1546                                          nb_mbuf_per_pool,
1547                                          socket_num == UMA_NO_CONFIG ?
1548                                          0 : socket_num, i);
1549         }
1550
1551         init_port_config();
1552
1553         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1554                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1555         /*
1556          * Records which Mbuf pool to use by each logical core, if needed.
1557          */
1558         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1559                 mbp = mbuf_pool_find(
1560                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1561
1562                 if (mbp == NULL)
1563                         mbp = mbuf_pool_find(0, 0);
1564                 fwd_lcores[lc_id]->mbp = mbp;
1565                 /* initialize GSO context */
1566                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1567                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1568                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1569                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1570                         RTE_ETHER_CRC_LEN;
1571                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1572         }
1573
1574         /* Configuration of packet forwarding streams. */
1575         if (init_fwd_streams() < 0)
1576                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1577
1578         fwd_config_setup();
1579
1580         /* create a gro context for each lcore */
1581         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1582         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1583         gro_param.max_item_per_flow = MAX_PKT_BURST;
1584         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1585                 gro_param.socket_id = rte_lcore_to_socket_id(
1586                                 fwd_lcores_cpuids[lc_id]);
1587                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1588                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1589                         rte_exit(EXIT_FAILURE,
1590                                         "rte_gro_ctx_create() failed\n");
1591                 }
1592         }
1593 }
1594
1595
1596 void
1597 reconfig(portid_t new_port_id, unsigned socket_id)
1598 {
1599         struct rte_port *port;
1600         int ret;
1601
1602         /* Reconfiguration of Ethernet ports. */
1603         port = &ports[new_port_id];
1604
1605         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1606         if (ret != 0)
1607                 return;
1608
1609         /* set flag to initialize port/queue */
1610         port->need_reconfig = 1;
1611         port->need_reconfig_queues = 1;
1612         port->socket_id = socket_id;
1613
1614         init_port_config();
1615 }
1616
1617
1618 int
1619 init_fwd_streams(void)
1620 {
1621         portid_t pid;
1622         struct rte_port *port;
1623         streamid_t sm_id, nb_fwd_streams_new;
1624         queueid_t q;
1625
1626         /* set socket id according to numa or not */
1627         RTE_ETH_FOREACH_DEV(pid) {
1628                 port = &ports[pid];
1629                 if (nb_rxq > port->dev_info.max_rx_queues) {
1630                         printf("Fail: nb_rxq(%d) is greater than "
1631                                 "max_rx_queues(%d)\n", nb_rxq,
1632                                 port->dev_info.max_rx_queues);
1633                         return -1;
1634                 }
1635                 if (nb_txq > port->dev_info.max_tx_queues) {
1636                         printf("Fail: nb_txq(%d) is greater than "
1637                                 "max_tx_queues(%d)\n", nb_txq,
1638                                 port->dev_info.max_tx_queues);
1639                         return -1;
1640                 }
1641                 if (numa_support) {
1642                         if (port_numa[pid] != NUMA_NO_CONFIG)
1643                                 port->socket_id = port_numa[pid];
1644                         else {
1645                                 port->socket_id = rte_eth_dev_socket_id(pid);
1646
1647                                 /*
1648                                  * if socket_id is invalid,
1649                                  * set to the first available socket.
1650                                  */
1651                                 if (check_socket_id(port->socket_id) < 0)
1652                                         port->socket_id = socket_ids[0];
1653                         }
1654                 }
1655                 else {
1656                         if (socket_num == UMA_NO_CONFIG)
1657                                 port->socket_id = 0;
1658                         else
1659                                 port->socket_id = socket_num;
1660                 }
1661         }
1662
1663         q = RTE_MAX(nb_rxq, nb_txq);
1664         if (q == 0) {
1665                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1666                 return -1;
1667         }
1668         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1669         if (nb_fwd_streams_new == nb_fwd_streams)
1670                 return 0;
1671         /* clear the old */
1672         if (fwd_streams != NULL) {
1673                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1674                         if (fwd_streams[sm_id] == NULL)
1675                                 continue;
1676                         rte_free(fwd_streams[sm_id]);
1677                         fwd_streams[sm_id] = NULL;
1678                 }
1679                 rte_free(fwd_streams);
1680                 fwd_streams = NULL;
1681         }
1682
1683         /* init new */
1684         nb_fwd_streams = nb_fwd_streams_new;
1685         if (nb_fwd_streams) {
1686                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1687                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1688                         RTE_CACHE_LINE_SIZE);
1689                 if (fwd_streams == NULL)
1690                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1691                                  " (struct fwd_stream *)) failed\n",
1692                                  nb_fwd_streams);
1693
1694                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1695                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1696                                 " struct fwd_stream", sizeof(struct fwd_stream),
1697                                 RTE_CACHE_LINE_SIZE);
1698                         if (fwd_streams[sm_id] == NULL)
1699                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1700                                          "(struct fwd_stream) failed\n");
1701                 }
1702         }
1703
1704         return 0;
1705 }
1706
1707 static void
1708 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1709 {
1710         uint64_t total_burst, sburst;
1711         uint64_t nb_burst;
1712         uint64_t burst_stats[4];
1713         uint16_t pktnb_stats[4];
1714         uint16_t nb_pkt;
1715         int burst_percent[4], sburstp;
1716         int i;
1717
1718         /*
1719          * First compute the total number of packet bursts and the
1720          * two highest numbers of bursts of the same number of packets.
1721          */
1722         memset(&burst_stats, 0x0, sizeof(burst_stats));
1723         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1724
1725         /* Show stats for 0 burst size always */
1726         total_burst = pbs->pkt_burst_spread[0];
1727         burst_stats[0] = pbs->pkt_burst_spread[0];
1728         pktnb_stats[0] = 0;
1729
1730         /* Find the next 2 burst sizes with highest occurrences. */
1731         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1732                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1733
1734                 if (nb_burst == 0)
1735                         continue;
1736
1737                 total_burst += nb_burst;
1738
1739                 if (nb_burst > burst_stats[1]) {
1740                         burst_stats[2] = burst_stats[1];
1741                         pktnb_stats[2] = pktnb_stats[1];
1742                         burst_stats[1] = nb_burst;
1743                         pktnb_stats[1] = nb_pkt;
1744                 } else if (nb_burst > burst_stats[2]) {
1745                         burst_stats[2] = nb_burst;
1746                         pktnb_stats[2] = nb_pkt;
1747                 }
1748         }
1749         if (total_burst == 0)
1750                 return;
1751
1752         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1753         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1754                 if (i == 3) {
1755                         printf("%d%% of other]\n", 100 - sburstp);
1756                         return;
1757                 }
1758
1759                 sburst += burst_stats[i];
1760                 if (sburst == total_burst) {
1761                         printf("%d%% of %d pkts]\n",
1762                                 100 - sburstp, (int) pktnb_stats[i]);
1763                         return;
1764                 }
1765
1766                 burst_percent[i] =
1767                         (double)burst_stats[i] / total_burst * 100;
1768                 printf("%d%% of %d pkts + ",
1769                         burst_percent[i], (int) pktnb_stats[i]);
1770                 sburstp += burst_percent[i];
1771         }
1772 }
1773
1774 static void
1775 fwd_stream_stats_display(streamid_t stream_id)
1776 {
1777         struct fwd_stream *fs;
1778         static const char *fwd_top_stats_border = "-------";
1779
1780         fs = fwd_streams[stream_id];
1781         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1782             (fs->fwd_dropped == 0))
1783                 return;
1784         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1785                "TX Port=%2d/Queue=%2d %s\n",
1786                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1787                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1788         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1789                " TX-dropped: %-14"PRIu64,
1790                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1791
1792         /* if checksum mode */
1793         if (cur_fwd_eng == &csum_fwd_engine) {
1794                 printf("  RX- bad IP checksum: %-14"PRIu64
1795                        "  Rx- bad L4 checksum: %-14"PRIu64
1796                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1797                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1798                         fs->rx_bad_outer_l4_csum);
1799         } else {
1800                 printf("\n");
1801         }
1802
1803         if (record_burst_stats) {
1804                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1805                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1806         }
1807 }
1808
1809 void
1810 fwd_stats_display(void)
1811 {
1812         static const char *fwd_stats_border = "----------------------";
1813         static const char *acc_stats_border = "+++++++++++++++";
1814         struct {
1815                 struct fwd_stream *rx_stream;
1816                 struct fwd_stream *tx_stream;
1817                 uint64_t tx_dropped;
1818                 uint64_t rx_bad_ip_csum;
1819                 uint64_t rx_bad_l4_csum;
1820                 uint64_t rx_bad_outer_l4_csum;
1821         } ports_stats[RTE_MAX_ETHPORTS];
1822         uint64_t total_rx_dropped = 0;
1823         uint64_t total_tx_dropped = 0;
1824         uint64_t total_rx_nombuf = 0;
1825         struct rte_eth_stats stats;
1826         uint64_t fwd_cycles = 0;
1827         uint64_t total_recv = 0;
1828         uint64_t total_xmit = 0;
1829         struct rte_port *port;
1830         streamid_t sm_id;
1831         portid_t pt_id;
1832         int i;
1833
1834         memset(ports_stats, 0, sizeof(ports_stats));
1835
1836         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1837                 struct fwd_stream *fs = fwd_streams[sm_id];
1838
1839                 if (cur_fwd_config.nb_fwd_streams >
1840                     cur_fwd_config.nb_fwd_ports) {
1841                         fwd_stream_stats_display(sm_id);
1842                 } else {
1843                         ports_stats[fs->tx_port].tx_stream = fs;
1844                         ports_stats[fs->rx_port].rx_stream = fs;
1845                 }
1846
1847                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1848
1849                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1850                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1851                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1852                                 fs->rx_bad_outer_l4_csum;
1853
1854                 if (record_core_cycles)
1855                         fwd_cycles += fs->core_cycles;
1856         }
1857         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1858                 pt_id = fwd_ports_ids[i];
1859                 port = &ports[pt_id];
1860
1861                 rte_eth_stats_get(pt_id, &stats);
1862                 stats.ipackets -= port->stats.ipackets;
1863                 stats.opackets -= port->stats.opackets;
1864                 stats.ibytes -= port->stats.ibytes;
1865                 stats.obytes -= port->stats.obytes;
1866                 stats.imissed -= port->stats.imissed;
1867                 stats.oerrors -= port->stats.oerrors;
1868                 stats.rx_nombuf -= port->stats.rx_nombuf;
1869
1870                 total_recv += stats.ipackets;
1871                 total_xmit += stats.opackets;
1872                 total_rx_dropped += stats.imissed;
1873                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1874                 total_tx_dropped += stats.oerrors;
1875                 total_rx_nombuf  += stats.rx_nombuf;
1876
1877                 printf("\n  %s Forward statistics for port %-2d %s\n",
1878                        fwd_stats_border, pt_id, fwd_stats_border);
1879
1880                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1881                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1882                        stats.ipackets + stats.imissed);
1883
1884                 if (cur_fwd_eng == &csum_fwd_engine)
1885                         printf("  Bad-ipcsum: %-14"PRIu64
1886                                " Bad-l4csum: %-14"PRIu64
1887                                "Bad-outer-l4csum: %-14"PRIu64"\n",
1888                                ports_stats[pt_id].rx_bad_ip_csum,
1889                                ports_stats[pt_id].rx_bad_l4_csum,
1890                                ports_stats[pt_id].rx_bad_outer_l4_csum);
1891                 if (stats.ierrors + stats.rx_nombuf > 0) {
1892                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
1893                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1894                 }
1895
1896                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1897                        "TX-total: %-"PRIu64"\n",
1898                        stats.opackets, ports_stats[pt_id].tx_dropped,
1899                        stats.opackets + ports_stats[pt_id].tx_dropped);
1900
1901                 if (record_burst_stats) {
1902                         if (ports_stats[pt_id].rx_stream)
1903                                 pkt_burst_stats_display("RX",
1904                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1905                         if (ports_stats[pt_id].tx_stream)
1906                                 pkt_burst_stats_display("TX",
1907                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1908                 }
1909
1910                 printf("  %s--------------------------------%s\n",
1911                        fwd_stats_border, fwd_stats_border);
1912         }
1913
1914         printf("\n  %s Accumulated forward statistics for all ports"
1915                "%s\n",
1916                acc_stats_border, acc_stats_border);
1917         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1918                "%-"PRIu64"\n"
1919                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1920                "%-"PRIu64"\n",
1921                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1922                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1923         if (total_rx_nombuf > 0)
1924                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1925         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1926                "%s\n",
1927                acc_stats_border, acc_stats_border);
1928         if (record_core_cycles) {
1929 #define CYC_PER_MHZ 1E6
1930                 if (total_recv > 0 || total_xmit > 0) {
1931                         uint64_t total_pkts = 0;
1932                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1933                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1934                                 total_pkts = total_xmit;
1935                         else
1936                                 total_pkts = total_recv;
1937
1938                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1939                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1940                                " MHz Clock\n",
1941                                (double) fwd_cycles / total_pkts,
1942                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1943                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1944                 }
1945         }
1946 }
1947
1948 void
1949 fwd_stats_reset(void)
1950 {
1951         streamid_t sm_id;
1952         portid_t pt_id;
1953         int i;
1954
1955         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1956                 pt_id = fwd_ports_ids[i];
1957                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1958         }
1959         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1960                 struct fwd_stream *fs = fwd_streams[sm_id];
1961
1962                 fs->rx_packets = 0;
1963                 fs->tx_packets = 0;
1964                 fs->fwd_dropped = 0;
1965                 fs->rx_bad_ip_csum = 0;
1966                 fs->rx_bad_l4_csum = 0;
1967                 fs->rx_bad_outer_l4_csum = 0;
1968
1969                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1970                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1971                 fs->core_cycles = 0;
1972         }
1973 }
1974
1975 static void
1976 flush_fwd_rx_queues(void)
1977 {
1978         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1979         portid_t  rxp;
1980         portid_t port_id;
1981         queueid_t rxq;
1982         uint16_t  nb_rx;
1983         uint16_t  i;
1984         uint8_t   j;
1985         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1986         uint64_t timer_period;
1987
1988         /* convert to number of cycles */
1989         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1990
1991         for (j = 0; j < 2; j++) {
1992                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1993                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1994                                 port_id = fwd_ports_ids[rxp];
1995                                 /**
1996                                 * testpmd can stuck in the below do while loop
1997                                 * if rte_eth_rx_burst() always returns nonzero
1998                                 * packets. So timer is added to exit this loop
1999                                 * after 1sec timer expiry.
2000                                 */
2001                                 prev_tsc = rte_rdtsc();
2002                                 do {
2003                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2004                                                 pkts_burst, MAX_PKT_BURST);
2005                                         for (i = 0; i < nb_rx; i++)
2006                                                 rte_pktmbuf_free(pkts_burst[i]);
2007
2008                                         cur_tsc = rte_rdtsc();
2009                                         diff_tsc = cur_tsc - prev_tsc;
2010                                         timer_tsc += diff_tsc;
2011                                 } while ((nb_rx > 0) &&
2012                                         (timer_tsc < timer_period));
2013                                 timer_tsc = 0;
2014                         }
2015                 }
2016                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2017         }
2018 }
2019
2020 static void
2021 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2022 {
2023         struct fwd_stream **fsm;
2024         streamid_t nb_fs;
2025         streamid_t sm_id;
2026 #ifdef RTE_LIB_BITRATESTATS
2027         uint64_t tics_per_1sec;
2028         uint64_t tics_datum;
2029         uint64_t tics_current;
2030         uint16_t i, cnt_ports;
2031
2032         cnt_ports = nb_ports;
2033         tics_datum = rte_rdtsc();
2034         tics_per_1sec = rte_get_timer_hz();
2035 #endif
2036         fsm = &fwd_streams[fc->stream_idx];
2037         nb_fs = fc->stream_nb;
2038         do {
2039                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2040                         (*pkt_fwd)(fsm[sm_id]);
2041 #ifdef RTE_LIB_BITRATESTATS
2042                 if (bitrate_enabled != 0 &&
2043                                 bitrate_lcore_id == rte_lcore_id()) {
2044                         tics_current = rte_rdtsc();
2045                         if (tics_current - tics_datum >= tics_per_1sec) {
2046                                 /* Periodic bitrate calculation */
2047                                 for (i = 0; i < cnt_ports; i++)
2048                                         rte_stats_bitrate_calc(bitrate_data,
2049                                                 ports_ids[i]);
2050                                 tics_datum = tics_current;
2051                         }
2052                 }
2053 #endif
2054 #ifdef RTE_LIB_LATENCYSTATS
2055                 if (latencystats_enabled != 0 &&
2056                                 latencystats_lcore_id == rte_lcore_id())
2057                         rte_latencystats_update();
2058 #endif
2059
2060         } while (! fc->stopped);
2061 }
2062
2063 static int
2064 start_pkt_forward_on_core(void *fwd_arg)
2065 {
2066         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2067                              cur_fwd_config.fwd_eng->packet_fwd);
2068         return 0;
2069 }
2070
2071 /*
2072  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2073  * Used to start communication flows in network loopback test configurations.
2074  */
2075 static int
2076 run_one_txonly_burst_on_core(void *fwd_arg)
2077 {
2078         struct fwd_lcore *fwd_lc;
2079         struct fwd_lcore tmp_lcore;
2080
2081         fwd_lc = (struct fwd_lcore *) fwd_arg;
2082         tmp_lcore = *fwd_lc;
2083         tmp_lcore.stopped = 1;
2084         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2085         return 0;
2086 }
2087
2088 /*
2089  * Launch packet forwarding:
2090  *     - Setup per-port forwarding context.
2091  *     - launch logical cores with their forwarding configuration.
2092  */
2093 static void
2094 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2095 {
2096         port_fwd_begin_t port_fwd_begin;
2097         unsigned int i;
2098         unsigned int lc_id;
2099         int diag;
2100
2101         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2102         if (port_fwd_begin != NULL) {
2103                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2104                         (*port_fwd_begin)(fwd_ports_ids[i]);
2105         }
2106         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2107                 lc_id = fwd_lcores_cpuids[i];
2108                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2109                         fwd_lcores[i]->stopped = 0;
2110                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2111                                                      fwd_lcores[i], lc_id);
2112                         if (diag != 0)
2113                                 printf("launch lcore %u failed - diag=%d\n",
2114                                        lc_id, diag);
2115                 }
2116         }
2117 }
2118
2119 /*
2120  * Launch packet forwarding configuration.
2121  */
2122 void
2123 start_packet_forwarding(int with_tx_first)
2124 {
2125         port_fwd_begin_t port_fwd_begin;
2126         port_fwd_end_t  port_fwd_end;
2127         struct rte_port *port;
2128         unsigned int i;
2129         portid_t   pt_id;
2130
2131         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2132                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2133
2134         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2135                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2136
2137         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2138                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2139                 (!nb_rxq || !nb_txq))
2140                 rte_exit(EXIT_FAILURE,
2141                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2142                         cur_fwd_eng->fwd_mode_name);
2143
2144         if (all_ports_started() == 0) {
2145                 printf("Not all ports were started\n");
2146                 return;
2147         }
2148         if (test_done == 0) {
2149                 printf("Packet forwarding already started\n");
2150                 return;
2151         }
2152
2153
2154         if(dcb_test) {
2155                 for (i = 0; i < nb_fwd_ports; i++) {
2156                         pt_id = fwd_ports_ids[i];
2157                         port = &ports[pt_id];
2158                         if (!port->dcb_flag) {
2159                                 printf("In DCB mode, all forwarding ports must "
2160                                        "be configured in this mode.\n");
2161                                 return;
2162                         }
2163                 }
2164                 if (nb_fwd_lcores == 1) {
2165                         printf("In DCB mode,the nb forwarding cores "
2166                                "should be larger than 1.\n");
2167                         return;
2168                 }
2169         }
2170         test_done = 0;
2171
2172         fwd_config_setup();
2173
2174         if(!no_flush_rx)
2175                 flush_fwd_rx_queues();
2176
2177         pkt_fwd_config_display(&cur_fwd_config);
2178         rxtx_config_display();
2179
2180         fwd_stats_reset();
2181         if (with_tx_first) {
2182                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2183                 if (port_fwd_begin != NULL) {
2184                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2185                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2186                 }
2187                 while (with_tx_first--) {
2188                         launch_packet_forwarding(
2189                                         run_one_txonly_burst_on_core);
2190                         rte_eal_mp_wait_lcore();
2191                 }
2192                 port_fwd_end = tx_only_engine.port_fwd_end;
2193                 if (port_fwd_end != NULL) {
2194                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2195                                 (*port_fwd_end)(fwd_ports_ids[i]);
2196                 }
2197         }
2198         launch_packet_forwarding(start_pkt_forward_on_core);
2199 }
2200
2201 void
2202 stop_packet_forwarding(void)
2203 {
2204         port_fwd_end_t port_fwd_end;
2205         lcoreid_t lc_id;
2206         portid_t pt_id;
2207         int i;
2208
2209         if (test_done) {
2210                 printf("Packet forwarding not started\n");
2211                 return;
2212         }
2213         printf("Telling cores to stop...");
2214         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2215                 fwd_lcores[lc_id]->stopped = 1;
2216         printf("\nWaiting for lcores to finish...\n");
2217         rte_eal_mp_wait_lcore();
2218         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2219         if (port_fwd_end != NULL) {
2220                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2221                         pt_id = fwd_ports_ids[i];
2222                         (*port_fwd_end)(pt_id);
2223                 }
2224         }
2225
2226         fwd_stats_display();
2227
2228         printf("\nDone.\n");
2229         test_done = 1;
2230 }
2231
2232 void
2233 dev_set_link_up(portid_t pid)
2234 {
2235         if (rte_eth_dev_set_link_up(pid) < 0)
2236                 printf("\nSet link up fail.\n");
2237 }
2238
2239 void
2240 dev_set_link_down(portid_t pid)
2241 {
2242         if (rte_eth_dev_set_link_down(pid) < 0)
2243                 printf("\nSet link down fail.\n");
2244 }
2245
2246 static int
2247 all_ports_started(void)
2248 {
2249         portid_t pi;
2250         struct rte_port *port;
2251
2252         RTE_ETH_FOREACH_DEV(pi) {
2253                 port = &ports[pi];
2254                 /* Check if there is a port which is not started */
2255                 if ((port->port_status != RTE_PORT_STARTED) &&
2256                         (port->slave_flag == 0))
2257                         return 0;
2258         }
2259
2260         /* No port is not started */
2261         return 1;
2262 }
2263
2264 int
2265 port_is_stopped(portid_t port_id)
2266 {
2267         struct rte_port *port = &ports[port_id];
2268
2269         if ((port->port_status != RTE_PORT_STOPPED) &&
2270             (port->slave_flag == 0))
2271                 return 0;
2272         return 1;
2273 }
2274
2275 int
2276 all_ports_stopped(void)
2277 {
2278         portid_t pi;
2279
2280         RTE_ETH_FOREACH_DEV(pi) {
2281                 if (!port_is_stopped(pi))
2282                         return 0;
2283         }
2284
2285         return 1;
2286 }
2287
2288 int
2289 port_is_started(portid_t port_id)
2290 {
2291         if (port_id_is_invalid(port_id, ENABLED_WARN))
2292                 return 0;
2293
2294         if (ports[port_id].port_status != RTE_PORT_STARTED)
2295                 return 0;
2296
2297         return 1;
2298 }
2299
2300 /* Configure the Rx and Tx hairpin queues for the selected port. */
2301 static int
2302 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2303 {
2304         queueid_t qi;
2305         struct rte_eth_hairpin_conf hairpin_conf = {
2306                 .peer_count = 1,
2307         };
2308         int i;
2309         int diag;
2310         struct rte_port *port = &ports[pi];
2311         uint16_t peer_rx_port = pi;
2312         uint16_t peer_tx_port = pi;
2313         uint32_t manual = 1;
2314         uint32_t tx_exp = hairpin_mode & 0x10;
2315
2316         if (!(hairpin_mode & 0xf)) {
2317                 peer_rx_port = pi;
2318                 peer_tx_port = pi;
2319                 manual = 0;
2320         } else if (hairpin_mode & 0x1) {
2321                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2322                                                        RTE_ETH_DEV_NO_OWNER);
2323                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2324                         peer_tx_port = rte_eth_find_next_owned_by(0,
2325                                                 RTE_ETH_DEV_NO_OWNER);
2326                 if (p_pi != RTE_MAX_ETHPORTS) {
2327                         peer_rx_port = p_pi;
2328                 } else {
2329                         uint16_t next_pi;
2330
2331                         /* Last port will be the peer RX port of the first. */
2332                         RTE_ETH_FOREACH_DEV(next_pi)
2333                                 peer_rx_port = next_pi;
2334                 }
2335                 manual = 1;
2336         } else if (hairpin_mode & 0x2) {
2337                 if (cnt_pi & 0x1) {
2338                         peer_rx_port = p_pi;
2339                 } else {
2340                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2341                                                 RTE_ETH_DEV_NO_OWNER);
2342                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2343                                 peer_rx_port = pi;
2344                 }
2345                 peer_tx_port = peer_rx_port;
2346                 manual = 1;
2347         }
2348
2349         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2350                 hairpin_conf.peers[0].port = peer_rx_port;
2351                 hairpin_conf.peers[0].queue = i + nb_rxq;
2352                 hairpin_conf.manual_bind = !!manual;
2353                 hairpin_conf.tx_explicit = !!tx_exp;
2354                 diag = rte_eth_tx_hairpin_queue_setup
2355                         (pi, qi, nb_txd, &hairpin_conf);
2356                 i++;
2357                 if (diag == 0)
2358                         continue;
2359
2360                 /* Fail to setup rx queue, return */
2361                 if (rte_atomic16_cmpset(&(port->port_status),
2362                                         RTE_PORT_HANDLING,
2363                                         RTE_PORT_STOPPED) == 0)
2364                         printf("Port %d can not be set back "
2365                                         "to stopped\n", pi);
2366                 printf("Fail to configure port %d hairpin "
2367                                 "queues\n", pi);
2368                 /* try to reconfigure queues next time */
2369                 port->need_reconfig_queues = 1;
2370                 return -1;
2371         }
2372         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2373                 hairpin_conf.peers[0].port = peer_tx_port;
2374                 hairpin_conf.peers[0].queue = i + nb_txq;
2375                 hairpin_conf.manual_bind = !!manual;
2376                 hairpin_conf.tx_explicit = !!tx_exp;
2377                 diag = rte_eth_rx_hairpin_queue_setup
2378                         (pi, qi, nb_rxd, &hairpin_conf);
2379                 i++;
2380                 if (diag == 0)
2381                         continue;
2382
2383                 /* Fail to setup rx queue, return */
2384                 if (rte_atomic16_cmpset(&(port->port_status),
2385                                         RTE_PORT_HANDLING,
2386                                         RTE_PORT_STOPPED) == 0)
2387                         printf("Port %d can not be set back "
2388                                         "to stopped\n", pi);
2389                 printf("Fail to configure port %d hairpin "
2390                                 "queues\n", pi);
2391                 /* try to reconfigure queues next time */
2392                 port->need_reconfig_queues = 1;
2393                 return -1;
2394         }
2395         return 0;
2396 }
2397
2398 /* Configure the Rx with optional split. */
2399 int
2400 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2401                uint16_t nb_rx_desc, unsigned int socket_id,
2402                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2403 {
2404         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2405         unsigned int i, mp_n;
2406         int ret;
2407
2408         if (rx_pkt_nb_segs <= 1 ||
2409             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2410                 rx_conf->rx_seg = NULL;
2411                 rx_conf->rx_nseg = 0;
2412                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2413                                              nb_rx_desc, socket_id,
2414                                              rx_conf, mp);
2415                 return ret;
2416         }
2417         for (i = 0; i < rx_pkt_nb_segs; i++) {
2418                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2419                 struct rte_mempool *mpx;
2420                 /*
2421                  * Use last valid pool for the segments with number
2422                  * exceeding the pool index.
2423                  */
2424                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2425                 mpx = mbuf_pool_find(socket_id, mp_n);
2426                 /* Handle zero as mbuf data buffer size. */
2427                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2428                                    rx_pkt_seg_lengths[i] :
2429                                    mbuf_data_size[mp_n];
2430                 rx_seg->offset = i < rx_pkt_nb_offs ?
2431                                    rx_pkt_seg_offsets[i] : 0;
2432                 rx_seg->mp = mpx ? mpx : mp;
2433         }
2434         rx_conf->rx_nseg = rx_pkt_nb_segs;
2435         rx_conf->rx_seg = rx_useg;
2436         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2437                                     socket_id, rx_conf, NULL);
2438         rx_conf->rx_seg = NULL;
2439         rx_conf->rx_nseg = 0;
2440         return ret;
2441 }
2442
2443 int
2444 start_port(portid_t pid)
2445 {
2446         int diag, need_check_link_status = -1;
2447         portid_t pi;
2448         portid_t p_pi = RTE_MAX_ETHPORTS;
2449         portid_t pl[RTE_MAX_ETHPORTS];
2450         portid_t peer_pl[RTE_MAX_ETHPORTS];
2451         uint16_t cnt_pi = 0;
2452         uint16_t cfg_pi = 0;
2453         int peer_pi;
2454         queueid_t qi;
2455         struct rte_port *port;
2456         struct rte_ether_addr mac_addr;
2457         struct rte_eth_hairpin_cap cap;
2458
2459         if (port_id_is_invalid(pid, ENABLED_WARN))
2460                 return 0;
2461
2462         if(dcb_config)
2463                 dcb_test = 1;
2464         RTE_ETH_FOREACH_DEV(pi) {
2465                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2466                         continue;
2467
2468                 need_check_link_status = 0;
2469                 port = &ports[pi];
2470                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2471                                                  RTE_PORT_HANDLING) == 0) {
2472                         printf("Port %d is now not stopped\n", pi);
2473                         continue;
2474                 }
2475
2476                 if (port->need_reconfig > 0) {
2477                         port->need_reconfig = 0;
2478
2479                         if (flow_isolate_all) {
2480                                 int ret = port_flow_isolate(pi, 1);
2481                                 if (ret) {
2482                                         printf("Failed to apply isolated"
2483                                                " mode on port %d\n", pi);
2484                                         return -1;
2485                                 }
2486                         }
2487                         configure_rxtx_dump_callbacks(0);
2488                         printf("Configuring Port %d (socket %u)\n", pi,
2489                                         port->socket_id);
2490                         if (nb_hairpinq > 0 &&
2491                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2492                                 printf("Port %d doesn't support hairpin "
2493                                        "queues\n", pi);
2494                                 return -1;
2495                         }
2496                         /* configure port */
2497                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2498                                                      nb_txq + nb_hairpinq,
2499                                                      &(port->dev_conf));
2500                         if (diag != 0) {
2501                                 if (rte_atomic16_cmpset(&(port->port_status),
2502                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2503                                         printf("Port %d can not be set back "
2504                                                         "to stopped\n", pi);
2505                                 printf("Fail to configure port %d\n", pi);
2506                                 /* try to reconfigure port next time */
2507                                 port->need_reconfig = 1;
2508                                 return -1;
2509                         }
2510                 }
2511                 if (port->need_reconfig_queues > 0) {
2512                         port->need_reconfig_queues = 0;
2513                         /* setup tx queues */
2514                         for (qi = 0; qi < nb_txq; qi++) {
2515                                 if ((numa_support) &&
2516                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2517                                         diag = rte_eth_tx_queue_setup(pi, qi,
2518                                                 port->nb_tx_desc[qi],
2519                                                 txring_numa[pi],
2520                                                 &(port->tx_conf[qi]));
2521                                 else
2522                                         diag = rte_eth_tx_queue_setup(pi, qi,
2523                                                 port->nb_tx_desc[qi],
2524                                                 port->socket_id,
2525                                                 &(port->tx_conf[qi]));
2526
2527                                 if (diag == 0)
2528                                         continue;
2529
2530                                 /* Fail to setup tx queue, return */
2531                                 if (rte_atomic16_cmpset(&(port->port_status),
2532                                                         RTE_PORT_HANDLING,
2533                                                         RTE_PORT_STOPPED) == 0)
2534                                         printf("Port %d can not be set back "
2535                                                         "to stopped\n", pi);
2536                                 printf("Fail to configure port %d tx queues\n",
2537                                        pi);
2538                                 /* try to reconfigure queues next time */
2539                                 port->need_reconfig_queues = 1;
2540                                 return -1;
2541                         }
2542                         for (qi = 0; qi < nb_rxq; qi++) {
2543                                 /* setup rx queues */
2544                                 if ((numa_support) &&
2545                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2546                                         struct rte_mempool * mp =
2547                                                 mbuf_pool_find
2548                                                         (rxring_numa[pi], 0);
2549                                         if (mp == NULL) {
2550                                                 printf("Failed to setup RX queue:"
2551                                                         "No mempool allocation"
2552                                                         " on the socket %d\n",
2553                                                         rxring_numa[pi]);
2554                                                 return -1;
2555                                         }
2556
2557                                         diag = rx_queue_setup(pi, qi,
2558                                              port->nb_rx_desc[qi],
2559                                              rxring_numa[pi],
2560                                              &(port->rx_conf[qi]),
2561                                              mp);
2562                                 } else {
2563                                         struct rte_mempool *mp =
2564                                                 mbuf_pool_find
2565                                                         (port->socket_id, 0);
2566                                         if (mp == NULL) {
2567                                                 printf("Failed to setup RX queue:"
2568                                                         "No mempool allocation"
2569                                                         " on the socket %d\n",
2570                                                         port->socket_id);
2571                                                 return -1;
2572                                         }
2573                                         diag = rx_queue_setup(pi, qi,
2574                                              port->nb_rx_desc[qi],
2575                                              port->socket_id,
2576                                              &(port->rx_conf[qi]),
2577                                              mp);
2578                                 }
2579                                 if (diag == 0)
2580                                         continue;
2581
2582                                 /* Fail to setup rx queue, return */
2583                                 if (rte_atomic16_cmpset(&(port->port_status),
2584                                                         RTE_PORT_HANDLING,
2585                                                         RTE_PORT_STOPPED) == 0)
2586                                         printf("Port %d can not be set back "
2587                                                         "to stopped\n", pi);
2588                                 printf("Fail to configure port %d rx queues\n",
2589                                        pi);
2590                                 /* try to reconfigure queues next time */
2591                                 port->need_reconfig_queues = 1;
2592                                 return -1;
2593                         }
2594                         /* setup hairpin queues */
2595                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2596                                 return -1;
2597                 }
2598                 configure_rxtx_dump_callbacks(verbose_level);
2599                 if (clear_ptypes) {
2600                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2601                                         NULL, 0);
2602                         if (diag < 0)
2603                                 printf(
2604                                 "Port %d: Failed to disable Ptype parsing\n",
2605                                 pi);
2606                 }
2607
2608                 p_pi = pi;
2609                 cnt_pi++;
2610
2611                 /* start port */
2612                 if (rte_eth_dev_start(pi) < 0) {
2613                         printf("Fail to start port %d\n", pi);
2614
2615                         /* Fail to setup rx queue, return */
2616                         if (rte_atomic16_cmpset(&(port->port_status),
2617                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2618                                 printf("Port %d can not be set back to "
2619                                                         "stopped\n", pi);
2620                         continue;
2621                 }
2622
2623                 if (rte_atomic16_cmpset(&(port->port_status),
2624                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2625                         printf("Port %d can not be set into started\n", pi);
2626
2627                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2628                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2629                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2630                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2631                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2632
2633                 /* at least one port started, need checking link status */
2634                 need_check_link_status = 1;
2635
2636                 pl[cfg_pi++] = pi;
2637         }
2638
2639         if (need_check_link_status == 1 && !no_link_check)
2640                 check_all_ports_link_status(RTE_PORT_ALL);
2641         else if (need_check_link_status == 0)
2642                 printf("Please stop the ports first\n");
2643
2644         if (hairpin_mode & 0xf) {
2645                 uint16_t i;
2646                 int j;
2647
2648                 /* bind all started hairpin ports */
2649                 for (i = 0; i < cfg_pi; i++) {
2650                         pi = pl[i];
2651                         /* bind current Tx to all peer Rx */
2652                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2653                                                         RTE_MAX_ETHPORTS, 1);
2654                         if (peer_pi < 0)
2655                                 return peer_pi;
2656                         for (j = 0; j < peer_pi; j++) {
2657                                 if (!port_is_started(peer_pl[j]))
2658                                         continue;
2659                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2660                                 if (diag < 0) {
2661                                         printf("Error during binding hairpin"
2662                                                " Tx port %u to %u: %s\n",
2663                                                pi, peer_pl[j],
2664                                                rte_strerror(-diag));
2665                                         return -1;
2666                                 }
2667                         }
2668                         /* bind all peer Tx to current Rx */
2669                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2670                                                         RTE_MAX_ETHPORTS, 0);
2671                         if (peer_pi < 0)
2672                                 return peer_pi;
2673                         for (j = 0; j < peer_pi; j++) {
2674                                 if (!port_is_started(peer_pl[j]))
2675                                         continue;
2676                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2677                                 if (diag < 0) {
2678                                         printf("Error during binding hairpin"
2679                                                " Tx port %u to %u: %s\n",
2680                                                peer_pl[j], pi,
2681                                                rte_strerror(-diag));
2682                                         return -1;
2683                                 }
2684                         }
2685                 }
2686         }
2687
2688         printf("Done\n");
2689         return 0;
2690 }
2691
2692 void
2693 stop_port(portid_t pid)
2694 {
2695         portid_t pi;
2696         struct rte_port *port;
2697         int need_check_link_status = 0;
2698         portid_t peer_pl[RTE_MAX_ETHPORTS];
2699         int peer_pi;
2700
2701         if (dcb_test) {
2702                 dcb_test = 0;
2703                 dcb_config = 0;
2704         }
2705
2706         if (port_id_is_invalid(pid, ENABLED_WARN))
2707                 return;
2708
2709         printf("Stopping ports...\n");
2710
2711         RTE_ETH_FOREACH_DEV(pi) {
2712                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2713                         continue;
2714
2715                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2716                         printf("Please remove port %d from forwarding configuration.\n", pi);
2717                         continue;
2718                 }
2719
2720                 if (port_is_bonding_slave(pi)) {
2721                         printf("Please remove port %d from bonded device.\n", pi);
2722                         continue;
2723                 }
2724
2725                 port = &ports[pi];
2726                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2727                                                 RTE_PORT_HANDLING) == 0)
2728                         continue;
2729
2730                 if (hairpin_mode & 0xf) {
2731                         int j;
2732
2733                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2734                         /* unbind all peer Tx from current Rx */
2735                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2736                                                         RTE_MAX_ETHPORTS, 0);
2737                         if (peer_pi < 0)
2738                                 continue;
2739                         for (j = 0; j < peer_pi; j++) {
2740                                 if (!port_is_started(peer_pl[j]))
2741                                         continue;
2742                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2743                         }
2744                 }
2745
2746                 if (port->flow_list)
2747                         port_flow_flush(pi);
2748
2749                 if (rte_eth_dev_stop(pi) != 0)
2750                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2751                                 pi);
2752
2753                 if (rte_atomic16_cmpset(&(port->port_status),
2754                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2755                         printf("Port %d can not be set into stopped\n", pi);
2756                 need_check_link_status = 1;
2757         }
2758         if (need_check_link_status && !no_link_check)
2759                 check_all_ports_link_status(RTE_PORT_ALL);
2760
2761         printf("Done\n");
2762 }
2763
2764 static void
2765 remove_invalid_ports_in(portid_t *array, portid_t *total)
2766 {
2767         portid_t i;
2768         portid_t new_total = 0;
2769
2770         for (i = 0; i < *total; i++)
2771                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2772                         array[new_total] = array[i];
2773                         new_total++;
2774                 }
2775         *total = new_total;
2776 }
2777
2778 static void
2779 remove_invalid_ports(void)
2780 {
2781         remove_invalid_ports_in(ports_ids, &nb_ports);
2782         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2783         nb_cfg_ports = nb_fwd_ports;
2784 }
2785
2786 void
2787 close_port(portid_t pid)
2788 {
2789         portid_t pi;
2790         struct rte_port *port;
2791
2792         if (port_id_is_invalid(pid, ENABLED_WARN))
2793                 return;
2794
2795         printf("Closing ports...\n");
2796
2797         RTE_ETH_FOREACH_DEV(pi) {
2798                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2799                         continue;
2800
2801                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2802                         printf("Please remove port %d from forwarding configuration.\n", pi);
2803                         continue;
2804                 }
2805
2806                 if (port_is_bonding_slave(pi)) {
2807                         printf("Please remove port %d from bonded device.\n", pi);
2808                         continue;
2809                 }
2810
2811                 port = &ports[pi];
2812                 if (rte_atomic16_cmpset(&(port->port_status),
2813                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2814                         printf("Port %d is already closed\n", pi);
2815                         continue;
2816                 }
2817
2818                 port_flow_flush(pi);
2819                 rte_eth_dev_close(pi);
2820         }
2821
2822         remove_invalid_ports();
2823         printf("Done\n");
2824 }
2825
2826 void
2827 reset_port(portid_t pid)
2828 {
2829         int diag;
2830         portid_t pi;
2831         struct rte_port *port;
2832
2833         if (port_id_is_invalid(pid, ENABLED_WARN))
2834                 return;
2835
2836         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2837                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2838                 printf("Can not reset port(s), please stop port(s) first.\n");
2839                 return;
2840         }
2841
2842         printf("Resetting ports...\n");
2843
2844         RTE_ETH_FOREACH_DEV(pi) {
2845                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2846                         continue;
2847
2848                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2849                         printf("Please remove port %d from forwarding "
2850                                "configuration.\n", pi);
2851                         continue;
2852                 }
2853
2854                 if (port_is_bonding_slave(pi)) {
2855                         printf("Please remove port %d from bonded device.\n",
2856                                pi);
2857                         continue;
2858                 }
2859
2860                 diag = rte_eth_dev_reset(pi);
2861                 if (diag == 0) {
2862                         port = &ports[pi];
2863                         port->need_reconfig = 1;
2864                         port->need_reconfig_queues = 1;
2865                 } else {
2866                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2867                 }
2868         }
2869
2870         printf("Done\n");
2871 }
2872
2873 void
2874 attach_port(char *identifier)
2875 {
2876         portid_t pi;
2877         struct rte_dev_iterator iterator;
2878
2879         printf("Attaching a new port...\n");
2880
2881         if (identifier == NULL) {
2882                 printf("Invalid parameters are specified\n");
2883                 return;
2884         }
2885
2886         if (rte_dev_probe(identifier) < 0) {
2887                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2888                 return;
2889         }
2890
2891         /* first attach mode: event */
2892         if (setup_on_probe_event) {
2893                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2894                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2895                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2896                                         ports[pi].need_setup != 0)
2897                                 setup_attached_port(pi);
2898                 return;
2899         }
2900
2901         /* second attach mode: iterator */
2902         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2903                 /* setup ports matching the devargs used for probing */
2904                 if (port_is_forwarding(pi))
2905                         continue; /* port was already attached before */
2906                 setup_attached_port(pi);
2907         }
2908 }
2909
2910 static void
2911 setup_attached_port(portid_t pi)
2912 {
2913         unsigned int socket_id;
2914         int ret;
2915
2916         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2917         /* if socket_id is invalid, set to the first available socket. */
2918         if (check_socket_id(socket_id) < 0)
2919                 socket_id = socket_ids[0];
2920         reconfig(pi, socket_id);
2921         ret = rte_eth_promiscuous_enable(pi);
2922         if (ret != 0)
2923                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2924                         pi, rte_strerror(-ret));
2925
2926         ports_ids[nb_ports++] = pi;
2927         fwd_ports_ids[nb_fwd_ports++] = pi;
2928         nb_cfg_ports = nb_fwd_ports;
2929         ports[pi].need_setup = 0;
2930         ports[pi].port_status = RTE_PORT_STOPPED;
2931
2932         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2933         printf("Done\n");
2934 }
2935
2936 static void
2937 detach_device(struct rte_device *dev)
2938 {
2939         portid_t sibling;
2940
2941         if (dev == NULL) {
2942                 printf("Device already removed\n");
2943                 return;
2944         }
2945
2946         printf("Removing a device...\n");
2947
2948         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2949                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2950                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2951                                 printf("Port %u not stopped\n", sibling);
2952                                 return;
2953                         }
2954                         port_flow_flush(sibling);
2955                 }
2956         }
2957
2958         if (rte_dev_remove(dev) < 0) {
2959                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2960                 return;
2961         }
2962         remove_invalid_ports();
2963
2964         printf("Device is detached\n");
2965         printf("Now total ports is %d\n", nb_ports);
2966         printf("Done\n");
2967         return;
2968 }
2969
2970 void
2971 detach_port_device(portid_t port_id)
2972 {
2973         if (port_id_is_invalid(port_id, ENABLED_WARN))
2974                 return;
2975
2976         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2977                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2978                         printf("Port not stopped\n");
2979                         return;
2980                 }
2981                 printf("Port was not closed\n");
2982         }
2983
2984         detach_device(rte_eth_devices[port_id].device);
2985 }
2986
2987 void
2988 detach_devargs(char *identifier)
2989 {
2990         struct rte_dev_iterator iterator;
2991         struct rte_devargs da;
2992         portid_t port_id;
2993
2994         printf("Removing a device...\n");
2995
2996         memset(&da, 0, sizeof(da));
2997         if (rte_devargs_parsef(&da, "%s", identifier)) {
2998                 printf("cannot parse identifier\n");
2999                 if (da.args)
3000                         free(da.args);
3001                 return;
3002         }
3003
3004         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3005                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3006                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3007                                 printf("Port %u not stopped\n", port_id);
3008                                 rte_eth_iterator_cleanup(&iterator);
3009                                 return;
3010                         }
3011                         port_flow_flush(port_id);
3012                 }
3013         }
3014
3015         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3016                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3017                             da.name, da.bus->name);
3018                 return;
3019         }
3020
3021         remove_invalid_ports();
3022
3023         printf("Device %s is detached\n", identifier);
3024         printf("Now total ports is %d\n", nb_ports);
3025         printf("Done\n");
3026 }
3027
3028 void
3029 pmd_test_exit(void)
3030 {
3031         portid_t pt_id;
3032         unsigned int i;
3033         int ret;
3034
3035         if (test_done == 0)
3036                 stop_packet_forwarding();
3037
3038         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3039                 if (mempools[i]) {
3040                         if (mp_alloc_type == MP_ALLOC_ANON)
3041                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3042                                                      NULL);
3043                 }
3044         }
3045         if (ports != NULL) {
3046                 no_link_check = 1;
3047                 RTE_ETH_FOREACH_DEV(pt_id) {
3048                         printf("\nStopping port %d...\n", pt_id);
3049                         fflush(stdout);
3050                         stop_port(pt_id);
3051                 }
3052                 RTE_ETH_FOREACH_DEV(pt_id) {
3053                         printf("\nShutting down port %d...\n", pt_id);
3054                         fflush(stdout);
3055                         close_port(pt_id);
3056                 }
3057         }
3058
3059         if (hot_plug) {
3060                 ret = rte_dev_event_monitor_stop();
3061                 if (ret) {
3062                         RTE_LOG(ERR, EAL,
3063                                 "fail to stop device event monitor.");
3064                         return;
3065                 }
3066
3067                 ret = rte_dev_event_callback_unregister(NULL,
3068                         dev_event_callback, NULL);
3069                 if (ret < 0) {
3070                         RTE_LOG(ERR, EAL,
3071                                 "fail to unregister device event callback.\n");
3072                         return;
3073                 }
3074
3075                 ret = rte_dev_hotplug_handle_disable();
3076                 if (ret) {
3077                         RTE_LOG(ERR, EAL,
3078                                 "fail to disable hotplug handling.\n");
3079                         return;
3080                 }
3081         }
3082         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3083                 if (mempools[i])
3084                         rte_mempool_free(mempools[i]);
3085         }
3086
3087         printf("\nBye...\n");
3088 }
3089
3090 typedef void (*cmd_func_t)(void);
3091 struct pmd_test_command {
3092         const char *cmd_name;
3093         cmd_func_t cmd_func;
3094 };
3095
3096 /* Check the link status of all ports in up to 9s, and print them finally */
3097 static void
3098 check_all_ports_link_status(uint32_t port_mask)
3099 {
3100 #define CHECK_INTERVAL 100 /* 100ms */
3101 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3102         portid_t portid;
3103         uint8_t count, all_ports_up, print_flag = 0;
3104         struct rte_eth_link link;
3105         int ret;
3106         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3107
3108         printf("Checking link statuses...\n");
3109         fflush(stdout);
3110         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3111                 all_ports_up = 1;
3112                 RTE_ETH_FOREACH_DEV(portid) {
3113                         if ((port_mask & (1 << portid)) == 0)
3114                                 continue;
3115                         memset(&link, 0, sizeof(link));
3116                         ret = rte_eth_link_get_nowait(portid, &link);
3117                         if (ret < 0) {
3118                                 all_ports_up = 0;
3119                                 if (print_flag == 1)
3120                                         printf("Port %u link get failed: %s\n",
3121                                                 portid, rte_strerror(-ret));
3122                                 continue;
3123                         }
3124                         /* print link status if flag set */
3125                         if (print_flag == 1) {
3126                                 rte_eth_link_to_str(link_status,
3127                                         sizeof(link_status), &link);
3128                                 printf("Port %d %s\n", portid, link_status);
3129                                 continue;
3130                         }
3131                         /* clear all_ports_up flag if any link down */
3132                         if (link.link_status == ETH_LINK_DOWN) {
3133                                 all_ports_up = 0;
3134                                 break;
3135                         }
3136                 }
3137                 /* after finally printing all link status, get out */
3138                 if (print_flag == 1)
3139                         break;
3140
3141                 if (all_ports_up == 0) {
3142                         fflush(stdout);
3143                         rte_delay_ms(CHECK_INTERVAL);
3144                 }
3145
3146                 /* set the print_flag if all ports up or timeout */
3147                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3148                         print_flag = 1;
3149                 }
3150
3151                 if (lsc_interrupt)
3152                         break;
3153         }
3154 }
3155
3156 static void
3157 rmv_port_callback(void *arg)
3158 {
3159         int need_to_start = 0;
3160         int org_no_link_check = no_link_check;
3161         portid_t port_id = (intptr_t)arg;
3162         struct rte_device *dev;
3163
3164         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3165
3166         if (!test_done && port_is_forwarding(port_id)) {
3167                 need_to_start = 1;
3168                 stop_packet_forwarding();
3169         }
3170         no_link_check = 1;
3171         stop_port(port_id);
3172         no_link_check = org_no_link_check;
3173
3174         /* Save rte_device pointer before closing ethdev port */
3175         dev = rte_eth_devices[port_id].device;
3176         close_port(port_id);
3177         detach_device(dev); /* might be already removed or have more ports */
3178
3179         if (need_to_start)
3180                 start_packet_forwarding(0);
3181 }
3182
3183 /* This function is used by the interrupt thread */
3184 static int
3185 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3186                   void *ret_param)
3187 {
3188         RTE_SET_USED(param);
3189         RTE_SET_USED(ret_param);
3190
3191         if (type >= RTE_ETH_EVENT_MAX) {
3192                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3193                         port_id, __func__, type);
3194                 fflush(stderr);
3195         } else if (event_print_mask & (UINT32_C(1) << type)) {
3196                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3197                         eth_event_desc[type]);
3198                 fflush(stdout);
3199         }
3200
3201         switch (type) {
3202         case RTE_ETH_EVENT_NEW:
3203                 ports[port_id].need_setup = 1;
3204                 ports[port_id].port_status = RTE_PORT_HANDLING;
3205                 break;
3206         case RTE_ETH_EVENT_INTR_RMV:
3207                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3208                         break;
3209                 if (rte_eal_alarm_set(100000,
3210                                 rmv_port_callback, (void *)(intptr_t)port_id))
3211                         fprintf(stderr, "Could not set up deferred device removal\n");
3212                 break;
3213         case RTE_ETH_EVENT_DESTROY:
3214                 ports[port_id].port_status = RTE_PORT_CLOSED;
3215                 printf("Port %u is closed\n", port_id);
3216                 break;
3217         default:
3218                 break;
3219         }
3220         return 0;
3221 }
3222
3223 static int
3224 register_eth_event_callback(void)
3225 {
3226         int ret;
3227         enum rte_eth_event_type event;
3228
3229         for (event = RTE_ETH_EVENT_UNKNOWN;
3230                         event < RTE_ETH_EVENT_MAX; event++) {
3231                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3232                                 event,
3233                                 eth_event_callback,
3234                                 NULL);
3235                 if (ret != 0) {
3236                         TESTPMD_LOG(ERR, "Failed to register callback for "
3237                                         "%s event\n", eth_event_desc[event]);
3238                         return -1;
3239                 }
3240         }
3241
3242         return 0;
3243 }
3244
3245 /* This function is used by the interrupt thread */
3246 static void
3247 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3248                              __rte_unused void *arg)
3249 {
3250         uint16_t port_id;
3251         int ret;
3252
3253         if (type >= RTE_DEV_EVENT_MAX) {
3254                 fprintf(stderr, "%s called upon invalid event %d\n",
3255                         __func__, type);
3256                 fflush(stderr);
3257         }
3258
3259         switch (type) {
3260         case RTE_DEV_EVENT_REMOVE:
3261                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3262                         device_name);
3263                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3264                 if (ret) {
3265                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3266                                 device_name);
3267                         return;
3268                 }
3269                 /*
3270                  * Because the user's callback is invoked in eal interrupt
3271                  * callback, the interrupt callback need to be finished before
3272                  * it can be unregistered when detaching device. So finish
3273                  * callback soon and use a deferred removal to detach device
3274                  * is need. It is a workaround, once the device detaching be
3275                  * moved into the eal in the future, the deferred removal could
3276                  * be deleted.
3277                  */
3278                 if (rte_eal_alarm_set(100000,
3279                                 rmv_port_callback, (void *)(intptr_t)port_id))
3280                         RTE_LOG(ERR, EAL,
3281                                 "Could not set up deferred device removal\n");
3282                 break;
3283         case RTE_DEV_EVENT_ADD:
3284                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3285                         device_name);
3286                 /* TODO: After finish kernel driver binding,
3287                  * begin to attach port.
3288                  */
3289                 break;
3290         default:
3291                 break;
3292         }
3293 }
3294
3295 static void
3296 rxtx_port_config(struct rte_port *port)
3297 {
3298         uint16_t qid;
3299         uint64_t offloads;
3300
3301         for (qid = 0; qid < nb_rxq; qid++) {
3302                 offloads = port->rx_conf[qid].offloads;
3303                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3304                 if (offloads != 0)
3305                         port->rx_conf[qid].offloads = offloads;
3306
3307                 /* Check if any Rx parameters have been passed */
3308                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3309                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3310
3311                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3312                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3313
3314                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3315                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3316
3317                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3318                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3319
3320                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3321                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3322
3323                 port->nb_rx_desc[qid] = nb_rxd;
3324         }
3325
3326         for (qid = 0; qid < nb_txq; qid++) {
3327                 offloads = port->tx_conf[qid].offloads;
3328                 port->tx_conf[qid] = port->dev_info.default_txconf;
3329                 if (offloads != 0)
3330                         port->tx_conf[qid].offloads = offloads;
3331
3332                 /* Check if any Tx parameters have been passed */
3333                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3334                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3335
3336                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3337                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3338
3339                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3340                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3341
3342                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3343                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3344
3345                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3346                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3347
3348                 port->nb_tx_desc[qid] = nb_txd;
3349         }
3350 }
3351
3352 /*
3353  * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3354  * MTU is also aligned if JUMBO_FRAME offload is not set.
3355  *
3356  * port->dev_info should be set before calling this function.
3357  *
3358  * return 0 on success, negative on error
3359  */
3360 int
3361 update_jumbo_frame_offload(portid_t portid)
3362 {
3363         struct rte_port *port = &ports[portid];
3364         uint32_t eth_overhead;
3365         uint64_t rx_offloads;
3366         int ret;
3367         bool on;
3368
3369         /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3370         if (port->dev_info.max_mtu != UINT16_MAX &&
3371             port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3372                 eth_overhead = port->dev_info.max_rx_pktlen -
3373                                 port->dev_info.max_mtu;
3374         else
3375                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3376
3377         rx_offloads = port->dev_conf.rxmode.offloads;
3378
3379         /* Default config value is 0 to use PMD specific overhead */
3380         if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3381                 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3382
3383         if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3384                 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3385                 on = false;
3386         } else {
3387                 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3388                         printf("Frame size (%u) is not supported by port %u\n",
3389                                 port->dev_conf.rxmode.max_rx_pkt_len,
3390                                 portid);
3391                         return -1;
3392                 }
3393                 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3394                 on = true;
3395         }
3396
3397         if (rx_offloads != port->dev_conf.rxmode.offloads) {
3398                 uint16_t qid;
3399
3400                 port->dev_conf.rxmode.offloads = rx_offloads;
3401
3402                 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3403                 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3404                         if (on)
3405                                 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3406                         else
3407                                 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3408                 }
3409         }
3410
3411         /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3412          * if unset do it here
3413          */
3414         if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3415                 ret = rte_eth_dev_set_mtu(portid,
3416                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3417                 if (ret)
3418                         printf("Failed to set MTU to %u for port %u\n",
3419                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3420                                 portid);
3421         }
3422
3423         return 0;
3424 }
3425
3426 void
3427 init_port_config(void)
3428 {
3429         portid_t pid;
3430         struct rte_port *port;
3431         int ret;
3432
3433         RTE_ETH_FOREACH_DEV(pid) {
3434                 port = &ports[pid];
3435                 port->dev_conf.fdir_conf = fdir_conf;
3436
3437                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3438                 if (ret != 0)
3439                         return;
3440
3441                 if (nb_rxq > 1) {
3442                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3443                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3444                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3445                 } else {
3446                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3447                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3448                 }
3449
3450                 if (port->dcb_flag == 0) {
3451                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3452                                 port->dev_conf.rxmode.mq_mode =
3453                                         (enum rte_eth_rx_mq_mode)
3454                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3455                         else
3456                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3457                 }
3458
3459                 rxtx_port_config(port);
3460
3461                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3462                 if (ret != 0)
3463                         return;
3464
3465 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3466                 rte_pmd_ixgbe_bypass_init(pid);
3467 #endif
3468
3469                 if (lsc_interrupt &&
3470                     (rte_eth_devices[pid].data->dev_flags &
3471                      RTE_ETH_DEV_INTR_LSC))
3472                         port->dev_conf.intr_conf.lsc = 1;
3473                 if (rmv_interrupt &&
3474                     (rte_eth_devices[pid].data->dev_flags &
3475                      RTE_ETH_DEV_INTR_RMV))
3476                         port->dev_conf.intr_conf.rmv = 1;
3477         }
3478 }
3479
3480 void set_port_slave_flag(portid_t slave_pid)
3481 {
3482         struct rte_port *port;
3483
3484         port = &ports[slave_pid];
3485         port->slave_flag = 1;
3486 }
3487
3488 void clear_port_slave_flag(portid_t slave_pid)
3489 {
3490         struct rte_port *port;
3491
3492         port = &ports[slave_pid];
3493         port->slave_flag = 0;
3494 }
3495
3496 uint8_t port_is_bonding_slave(portid_t slave_pid)
3497 {
3498         struct rte_port *port;
3499
3500         port = &ports[slave_pid];
3501         if ((rte_eth_devices[slave_pid].data->dev_flags &
3502             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3503                 return 1;
3504         return 0;
3505 }
3506
3507 const uint16_t vlan_tags[] = {
3508                 0,  1,  2,  3,  4,  5,  6,  7,
3509                 8,  9, 10, 11,  12, 13, 14, 15,
3510                 16, 17, 18, 19, 20, 21, 22, 23,
3511                 24, 25, 26, 27, 28, 29, 30, 31
3512 };
3513
3514 static  int
3515 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3516                  enum dcb_mode_enable dcb_mode,
3517                  enum rte_eth_nb_tcs num_tcs,
3518                  uint8_t pfc_en)
3519 {
3520         uint8_t i;
3521         int32_t rc;
3522         struct rte_eth_rss_conf rss_conf;
3523
3524         /*
3525          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3526          * given above, and the number of traffic classes available for use.
3527          */
3528         if (dcb_mode == DCB_VT_ENABLED) {
3529                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3530                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3531                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3532                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3533
3534                 /* VMDQ+DCB RX and TX configurations */
3535                 vmdq_rx_conf->enable_default_pool = 0;
3536                 vmdq_rx_conf->default_pool = 0;
3537                 vmdq_rx_conf->nb_queue_pools =
3538                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3539                 vmdq_tx_conf->nb_queue_pools =
3540                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3541
3542                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3543                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3544                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3545                         vmdq_rx_conf->pool_map[i].pools =
3546                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3547                 }
3548                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3549                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3550                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3551                 }
3552
3553                 /* set DCB mode of RX and TX of multiple queues */
3554                 eth_conf->rxmode.mq_mode =
3555                                 (enum rte_eth_rx_mq_mode)
3556                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3557                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3558         } else {
3559                 struct rte_eth_dcb_rx_conf *rx_conf =
3560                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3561                 struct rte_eth_dcb_tx_conf *tx_conf =
3562                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3563
3564                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3565
3566                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3567                 if (rc != 0)
3568                         return rc;
3569
3570                 rx_conf->nb_tcs = num_tcs;
3571                 tx_conf->nb_tcs = num_tcs;
3572
3573                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3574                         rx_conf->dcb_tc[i] = i % num_tcs;
3575                         tx_conf->dcb_tc[i] = i % num_tcs;
3576                 }
3577
3578                 eth_conf->rxmode.mq_mode =
3579                                 (enum rte_eth_rx_mq_mode)
3580                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3581                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3582                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3583         }
3584
3585         if (pfc_en)
3586                 eth_conf->dcb_capability_en =
3587                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3588         else
3589                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3590
3591         return 0;
3592 }
3593
3594 int
3595 init_port_dcb_config(portid_t pid,
3596                      enum dcb_mode_enable dcb_mode,
3597                      enum rte_eth_nb_tcs num_tcs,
3598                      uint8_t pfc_en)
3599 {
3600         struct rte_eth_conf port_conf;
3601         struct rte_port *rte_port;
3602         int retval;
3603         uint16_t i;
3604
3605         rte_port = &ports[pid];
3606
3607         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3608         /* Enter DCB configuration status */
3609         dcb_config = 1;
3610
3611         port_conf.rxmode = rte_port->dev_conf.rxmode;
3612         port_conf.txmode = rte_port->dev_conf.txmode;
3613
3614         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3615         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3616         if (retval < 0)
3617                 return retval;
3618         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3619
3620         /* re-configure the device . */
3621         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3622         if (retval < 0)
3623                 return retval;
3624
3625         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3626         if (retval != 0)
3627                 return retval;
3628
3629         /* If dev_info.vmdq_pool_base is greater than 0,
3630          * the queue id of vmdq pools is started after pf queues.
3631          */
3632         if (dcb_mode == DCB_VT_ENABLED &&
3633             rte_port->dev_info.vmdq_pool_base > 0) {
3634                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3635                         " for port %d.", pid);
3636                 return -1;
3637         }
3638
3639         /* Assume the ports in testpmd have the same dcb capability
3640          * and has the same number of rxq and txq in dcb mode
3641          */
3642         if (dcb_mode == DCB_VT_ENABLED) {
3643                 if (rte_port->dev_info.max_vfs > 0) {
3644                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3645                         nb_txq = rte_port->dev_info.nb_tx_queues;
3646                 } else {
3647                         nb_rxq = rte_port->dev_info.max_rx_queues;
3648                         nb_txq = rte_port->dev_info.max_tx_queues;
3649                 }
3650         } else {
3651                 /*if vt is disabled, use all pf queues */
3652                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3653                         nb_rxq = rte_port->dev_info.max_rx_queues;
3654                         nb_txq = rte_port->dev_info.max_tx_queues;
3655                 } else {
3656                         nb_rxq = (queueid_t)num_tcs;
3657                         nb_txq = (queueid_t)num_tcs;
3658
3659                 }
3660         }
3661         rx_free_thresh = 64;
3662
3663         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3664
3665         rxtx_port_config(rte_port);
3666         /* VLAN filter */
3667         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3668         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3669                 rx_vft_set(pid, vlan_tags[i], 1);
3670
3671         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3672         if (retval != 0)
3673                 return retval;
3674
3675         rte_port->dcb_flag = 1;
3676
3677         return 0;
3678 }
3679
3680 static void
3681 init_port(void)
3682 {
3683         int i;
3684
3685         /* Configuration of Ethernet ports. */
3686         ports = rte_zmalloc("testpmd: ports",
3687                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3688                             RTE_CACHE_LINE_SIZE);
3689         if (ports == NULL) {
3690                 rte_exit(EXIT_FAILURE,
3691                                 "rte_zmalloc(%d struct rte_port) failed\n",
3692                                 RTE_MAX_ETHPORTS);
3693         }
3694         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3695                 LIST_INIT(&ports[i].flow_tunnel_list);
3696         /* Initialize ports NUMA structures */
3697         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3698         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3699         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3700 }
3701
3702 static void
3703 force_quit(void)
3704 {
3705         pmd_test_exit();
3706         prompt_exit();
3707 }
3708
3709 static void
3710 print_stats(void)
3711 {
3712         uint8_t i;
3713         const char clr[] = { 27, '[', '2', 'J', '\0' };
3714         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3715
3716         /* Clear screen and move to top left */
3717         printf("%s%s", clr, top_left);
3718
3719         printf("\nPort statistics ====================================");
3720         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3721                 nic_stats_display(fwd_ports_ids[i]);
3722
3723         fflush(stdout);
3724 }
3725
3726 static void
3727 signal_handler(int signum)
3728 {
3729         if (signum == SIGINT || signum == SIGTERM) {
3730                 printf("\nSignal %d received, preparing to exit...\n",
3731                                 signum);
3732 #ifdef RTE_LIB_PDUMP
3733                 /* uninitialize packet capture framework */
3734                 rte_pdump_uninit();
3735 #endif
3736 #ifdef RTE_LIB_LATENCYSTATS
3737                 if (latencystats_enabled != 0)
3738                         rte_latencystats_uninit();
3739 #endif
3740                 force_quit();
3741                 /* Set flag to indicate the force termination. */
3742                 f_quit = 1;
3743                 /* exit with the expected status */
3744                 signal(signum, SIG_DFL);
3745                 kill(getpid(), signum);
3746         }
3747 }
3748
3749 int
3750 main(int argc, char** argv)
3751 {
3752         int diag;
3753         portid_t port_id;
3754         uint16_t count;
3755         int ret;
3756
3757         signal(SIGINT, signal_handler);
3758         signal(SIGTERM, signal_handler);
3759
3760         testpmd_logtype = rte_log_register("testpmd");
3761         if (testpmd_logtype < 0)
3762                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3763         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3764
3765         diag = rte_eal_init(argc, argv);
3766         if (diag < 0)
3767                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3768                          rte_strerror(rte_errno));
3769
3770         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3771                 rte_exit(EXIT_FAILURE,
3772                          "Secondary process type not supported.\n");
3773
3774         ret = register_eth_event_callback();
3775         if (ret != 0)
3776                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3777
3778 #ifdef RTE_LIB_PDUMP
3779         /* initialize packet capture framework */
3780         rte_pdump_init();
3781 #endif
3782
3783         count = 0;
3784         RTE_ETH_FOREACH_DEV(port_id) {
3785                 ports_ids[count] = port_id;
3786                 count++;
3787         }
3788         nb_ports = (portid_t) count;
3789         if (nb_ports == 0)
3790                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3791
3792         /* allocate port structures, and init them */
3793         init_port();
3794
3795         set_def_fwd_config();
3796         if (nb_lcores == 0)
3797                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3798                          "Check the core mask argument\n");
3799
3800         /* Bitrate/latency stats disabled by default */
3801 #ifdef RTE_LIB_BITRATESTATS
3802         bitrate_enabled = 0;
3803 #endif
3804 #ifdef RTE_LIB_LATENCYSTATS
3805         latencystats_enabled = 0;
3806 #endif
3807
3808         /* on FreeBSD, mlockall() is disabled by default */
3809 #ifdef RTE_EXEC_ENV_FREEBSD
3810         do_mlockall = 0;
3811 #else
3812         do_mlockall = 1;
3813 #endif
3814
3815         argc -= diag;
3816         argv += diag;
3817         if (argc > 1)
3818                 launch_args_parse(argc, argv);
3819
3820         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3821                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3822                         strerror(errno));
3823         }
3824
3825         if (tx_first && interactive)
3826                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3827                                 "interactive mode.\n");
3828
3829         if (tx_first && lsc_interrupt) {
3830                 printf("Warning: lsc_interrupt needs to be off when "
3831                                 " using tx_first. Disabling.\n");
3832                 lsc_interrupt = 0;
3833         }
3834
3835         if (!nb_rxq && !nb_txq)
3836                 printf("Warning: Either rx or tx queues should be non-zero\n");
3837
3838         if (nb_rxq > 1 && nb_rxq > nb_txq)
3839                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3840                        "but nb_txq=%d will prevent to fully test it.\n",
3841                        nb_rxq, nb_txq);
3842
3843         init_config();
3844
3845         if (hot_plug) {
3846                 ret = rte_dev_hotplug_handle_enable();
3847                 if (ret) {
3848                         RTE_LOG(ERR, EAL,
3849                                 "fail to enable hotplug handling.");
3850                         return -1;
3851                 }
3852
3853                 ret = rte_dev_event_monitor_start();
3854                 if (ret) {
3855                         RTE_LOG(ERR, EAL,
3856                                 "fail to start device event monitoring.");
3857                         return -1;
3858                 }
3859
3860                 ret = rte_dev_event_callback_register(NULL,
3861                         dev_event_callback, NULL);
3862                 if (ret) {
3863                         RTE_LOG(ERR, EAL,
3864                                 "fail  to register device event callback\n");
3865                         return -1;
3866                 }
3867         }
3868
3869         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3870                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3871
3872         /* set all ports to promiscuous mode by default */
3873         RTE_ETH_FOREACH_DEV(port_id) {
3874                 ret = rte_eth_promiscuous_enable(port_id);
3875                 if (ret != 0)
3876                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3877                                 port_id, rte_strerror(-ret));
3878         }
3879
3880         /* Init metrics library */
3881         rte_metrics_init(rte_socket_id());
3882
3883 #ifdef RTE_LIB_LATENCYSTATS
3884         if (latencystats_enabled != 0) {
3885                 int ret = rte_latencystats_init(1, NULL);
3886                 if (ret)
3887                         printf("Warning: latencystats init()"
3888                                 " returned error %d\n", ret);
3889                 printf("Latencystats running on lcore %d\n",
3890                         latencystats_lcore_id);
3891         }
3892 #endif
3893
3894         /* Setup bitrate stats */
3895 #ifdef RTE_LIB_BITRATESTATS
3896         if (bitrate_enabled != 0) {
3897                 bitrate_data = rte_stats_bitrate_create();
3898                 if (bitrate_data == NULL)
3899                         rte_exit(EXIT_FAILURE,
3900                                 "Could not allocate bitrate data.\n");
3901                 rte_stats_bitrate_reg(bitrate_data);
3902         }
3903 #endif
3904
3905 #ifdef RTE_LIB_CMDLINE
3906         if (strlen(cmdline_filename) != 0)
3907                 cmdline_read_from_file(cmdline_filename);
3908
3909         if (interactive == 1) {
3910                 if (auto_start) {
3911                         printf("Start automatic packet forwarding\n");
3912                         start_packet_forwarding(0);
3913                 }
3914                 prompt();
3915                 pmd_test_exit();
3916         } else
3917 #endif
3918         {
3919                 char c;
3920                 int rc;
3921
3922                 f_quit = 0;
3923
3924                 printf("No commandline core given, start packet forwarding\n");
3925                 start_packet_forwarding(tx_first);
3926                 if (stats_period != 0) {
3927                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3928                         uint64_t timer_period;
3929
3930                         /* Convert to number of cycles */
3931                         timer_period = stats_period * rte_get_timer_hz();
3932
3933                         while (f_quit == 0) {
3934                                 cur_time = rte_get_timer_cycles();
3935                                 diff_time += cur_time - prev_time;
3936
3937                                 if (diff_time >= timer_period) {
3938                                         print_stats();
3939                                         /* Reset the timer */
3940                                         diff_time = 0;
3941                                 }
3942                                 /* Sleep to avoid unnecessary checks */
3943                                 prev_time = cur_time;
3944                                 sleep(1);
3945                         }
3946                 }
3947
3948                 printf("Press enter to exit\n");
3949                 rc = read(0, &c, 1);
3950                 pmd_test_exit();
3951                 if (rc < 0)
3952                         return 1;
3953         }
3954
3955         ret = rte_eal_cleanup();
3956         if (ret != 0)
3957                 rte_exit(EXIT_FAILURE,
3958                          "EAL cleanup failed: %s\n", strerror(-ret));
3959
3960         return EXIT_SUCCESS;
3961 }