app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #ifndef RTE_EXEC_ENV_WINDOWS
  13 #include <sys/mman.h>
  14 #endif
  15 #include <sys/types.h>
  16 #include <errno.h>
  17 #include <stdbool.h>
  18
  19 #include <sys/queue.h>
  20 #include <sys/stat.h>
  21
  22 #include <stdint.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_byteorder.h>
  29 #include <rte_log.h>
  30 #include <rte_debug.h>
  31 #include <rte_cycles.h>
  32 #include <rte_memory.h>
  33 #include <rte_memcpy.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_alarm.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_atomic.h>
  40 #include <rte_branch_prediction.h>
  41 #include <rte_mempool.h>
  42 #include <rte_malloc.h>
  43 #include <rte_mbuf.h>
  44 #include <rte_mbuf_pool_ops.h>
  45 #include <rte_interrupts.h>
  46 #include <rte_pci.h>
  47 #include <rte_ether.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_dev.h>
  50 #include <rte_string_fns.h>
  51 #ifdef RTE_NET_IXGBE
  52 #include <rte_pmd_ixgbe.h>
  53 #endif
  54 #ifdef RTE_LIB_PDUMP
  55 #include <rte_pdump.h>
  56 #endif
  57 #include <rte_flow.h>
  58 #include <rte_metrics.h>
  59 #ifdef RTE_LIB_BITRATESTATS
  60 #include <rte_bitrate.h>
  61 #endif
  62 #ifdef RTE_LIB_LATENCYSTATS
  63 #include <rte_latencystats.h>
  64 #endif
  65 #ifdef RTE_EXEC_ENV_WINDOWS
  66 #include <process.h>
  67 #endif
  68
  69 #include "testpmd.h"
  70
  71 #ifndef MAP_HUGETLB
  72 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  73 #define HUGE_FLAG (0x40000)
  74 #else
  75 #define HUGE_FLAG MAP_HUGETLB
  76 #endif
  77
  78 #ifndef MAP_HUGE_SHIFT
  79 /* older kernels (or FreeBSD) will not have this define */
  80 #define HUGE_SHIFT (26)
  81 #else
  82 #define HUGE_SHIFT MAP_HUGE_SHIFT
  83 #endif
  84
  85 #define EXTMEM_HEAP_NAME "extmem"
  86 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  87
  88 uint16_t verbose_level = 0; /**< Silent by default. */
  89 int testpmd_logtype; /**< Log type for testpmd logs */
  90
  91 /* use main core for command line ? */
  92 uint8_t interactive = 0;
  93 uint8_t auto_start = 0;
  94 uint8_t tx_first;
  95 char cmdline_filename[PATH_MAX] = {0};
  96
  97 /*
  98  * NUMA support configuration.
  99  * When set, the NUMA support attempts to dispatch the allocation of the
 100  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 101  * probed ports among the CPU sockets 0 and 1.
 102  * Otherwise, all memory is allocated from CPU socket 0.
 103  */
 104 uint8_t numa_support = 1; /**< numa enabled by default */
 105
 106 /*
 107  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 108  * not configured.
 109  */
 110 uint8_t socket_num = UMA_NO_CONFIG;
 111
 112 /*
 113  * Select mempool allocation type:
 114  * - native: use regular DPDK memory
 115  * - anon: use regular DPDK memory to create mempool, but populate using
 116  *         anonymous memory (may not be IOVA-contiguous)
 117  * - xmem: use externally allocated hugepage memory
 118  */
 119 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 120
 121 /*
 122  * Store specified sockets on which memory pool to be used by ports
 123  * is allocated.
 124  */
 125 uint8_t port_numa[RTE_MAX_ETHPORTS];
 126
 127 /*
 128  * Store specified sockets on which RX ring to be used by ports
 129  * is allocated.
 130  */
 131 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 132
 133 /*
 134  * Store specified sockets on which TX ring to be used by ports
 135  * is allocated.
 136  */
 137 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 138
 139 /*
 140  * Record the Ethernet address of peer target ports to which packets are
 141  * forwarded.
 142  * Must be instantiated with the ethernet addresses of peer traffic generator
 143  * ports.
 144  */
 145 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 146 portid_t nb_peer_eth_addrs = 0;
 147
 148 /*
 149  * Probed Target Environment.
 150  */
 151 struct rte_port *ports;        /**< For all probed ethernet ports. */
 152 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 153 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 154 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 155
 156 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 157
 158 /*
 159  * Test Forwarding Configuration.
 160  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 161  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 162  */
 163 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 164 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 165 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 166 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 167
 168 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 169 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 170
 171 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 172 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 173
 174 /*
 175  * Forwarding engines.
 176  */
 177 struct fwd_engine * fwd_engines[] = {
 178         &io_fwd_engine,
 179         &mac_fwd_engine,
 180         &mac_swap_engine,
 181         &flow_gen_engine,
 182         &rx_only_engine,
 183         &tx_only_engine,
 184         &csum_fwd_engine,
 185         &icmp_echo_engine,
 186         &noisy_vnf_engine,
 187         &five_tuple_swap_fwd_engine,
 188 #ifdef RTE_LIBRTE_IEEE1588
 189         &ieee1588_fwd_engine,
 190 #endif
 191         NULL,
 192 };
 193
 194 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 195 uint16_t mempool_flags;
 196
 197 struct fwd_config cur_fwd_config;
 198 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 199 uint32_t retry_enabled;
 200 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 201 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 202
 203 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 204 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 205         DEFAULT_MBUF_DATA_SIZE
 206 }; /**< Mbuf data space size. */
 207 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 208                                       * specified on command-line. */
 209 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 210
 211 /*
 212  * In container, it cannot terminate the process which running with 'stats-period'
 213  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 214  */
 215 uint8_t f_quit;
 216
 217 /*
 218  * Configuration of packet segments used to scatter received packets
 219  * if some of split features is configured.
 220  */
 221 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 222 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 223 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 224 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 225
 226 /*
 227  * Configuration of packet segments used by the "txonly" processing engine.
 228  */
 229 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 230 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 231         TXONLY_DEF_PACKET_LEN,
 232 };
 233 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 234
 235 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 236 /**< Split policy for packets to TX. */
 237
 238 uint8_t txonly_multi_flow;
 239 /**< Whether multiple flows are generated in TXONLY mode. */
 240
 241 uint32_t tx_pkt_times_inter;
 242 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 243
 244 uint32_t tx_pkt_times_intra;
 245 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 246
 247 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 248 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 249 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 250
 251 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 252 uint8_t dcb_config = 0;
 253
 254 /*
 255  * Configurable number of RX/TX queues.
 256  */
 257 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 258 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 259 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 260
 261 /*
 262  * Configurable number of RX/TX ring descriptors.
 263  * Defaults are supplied by drivers via ethdev.
 264  */
 265 #define RTE_TEST_RX_DESC_DEFAULT 0
 266 #define RTE_TEST_TX_DESC_DEFAULT 0
 267 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 268 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 269
 270 #define RTE_PMD_PARAM_UNSET -1
 271 /*
 272  * Configurable values of RX and TX ring threshold registers.
 273  */
 274
 275 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 276 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 277 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 278
 279 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 280 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 281 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of RX free threshold.
 285  */
 286 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 287
 288 /*
 289  * Configurable value of RX drop enable.
 290  */
 291 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 292
 293 /*
 294  * Configurable value of TX free threshold.
 295  */
 296 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 297
 298 /*
 299  * Configurable value of TX RS bit threshold.
 300  */
 301 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 302
 303 /*
 304  * Configurable value of buffered packets before sending.
 305  */
 306 uint16_t noisy_tx_sw_bufsz;
 307
 308 /*
 309  * Configurable value of packet buffer timeout.
 310  */
 311 uint16_t noisy_tx_sw_buf_flush_time;
 312
 313 /*
 314  * Configurable value for size of VNF internal memory area
 315  * used for simulating noisy neighbour behaviour
 316  */
 317 uint64_t noisy_lkup_mem_sz;
 318
 319 /*
 320  * Configurable value of number of random writes done in
 321  * VNF simulation memory area.
 322  */
 323 uint64_t noisy_lkup_num_writes;
 324
 325 /*
 326  * Configurable value of number of random reads done in
 327  * VNF simulation memory area.
 328  */
 329 uint64_t noisy_lkup_num_reads;
 330
 331 /*
 332  * Configurable value of number of random reads/writes done in
 333  * VNF simulation memory area.
 334  */
 335 uint64_t noisy_lkup_num_reads_writes;
 336
 337 /*
 338  * Receive Side Scaling (RSS) configuration.
 339  */
 340 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 341
 342 /*
 343  * Port topology configuration
 344  */
 345 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 346
 347 /*
 348  * Avoids to flush all the RX streams before starts forwarding.
 349  */
 350 uint8_t no_flush_rx = 0; /* flush by default */
 351
 352 /*
 353  * Flow API isolated mode.
 354  */
 355 uint8_t flow_isolate_all;
 356
 357 /*
 358  * Avoids to check link status when starting/stopping a port.
 359  */
 360 uint8_t no_link_check = 0; /* check by default */
 361
 362 /*
 363  * Don't automatically start all ports in interactive mode.
 364  */
 365 uint8_t no_device_start = 0;
 366
 367 /*
 368  * Enable link status change notification
 369  */
 370 uint8_t lsc_interrupt = 1; /* enabled by default */
 371
 372 /*
 373  * Enable device removal notification.
 374  */
 375 uint8_t rmv_interrupt = 1; /* enabled by default */
 376
 377 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 378
 379 /* After attach, port setup is called on event or by iterator */
 380 bool setup_on_probe_event = true;
 381
 382 /* Clear ptypes on port initialization. */
 383 uint8_t clear_ptypes = true;
 384
 385 /* Hairpin ports configuration mode. */
 386 uint16_t hairpin_mode;
 387
 388 /* Pretty printing of ethdev events */
 389 static const char * const eth_event_desc[] = {
 390         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 391         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 392         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 393         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 394         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 395         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 396         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 397         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 398         [RTE_ETH_EVENT_NEW] = "device probed",
 399         [RTE_ETH_EVENT_DESTROY] = "device released",
 400         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 401         [RTE_ETH_EVENT_MAX] = NULL,
 402 };
 403
 404 /*
 405  * Display or mask ether events
 406  * Default to all events except VF_MBOX
 407  */
 408 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 413                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 414                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 415                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 416 /*
 417  * Decide if all memory are locked for performance.
 418  */
 419 int do_mlockall = 0;
 420
 421 /*
 422  * NIC bypass mode configuration options.
 423  */
 424
 425 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 426 /* The NIC bypass watchdog timeout. */
 427 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 428 #endif
 429
 430
 431 #ifdef RTE_LIB_LATENCYSTATS
 432
 433 /*
 434  * Set when latency stats is enabled in the commandline
 435  */
 436 uint8_t latencystats_enabled;
 437
 438 /*
 439  * Lcore ID to serive latency statistics.
 440  */
 441 lcoreid_t latencystats_lcore_id = -1;
 442
 443 #endif
 444
 445 /*
 446  * Ethernet device configuration.
 447  */
 448 struct rte_eth_rxmode rx_mode = {
 449         /* Default maximum frame length.
 450          * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
 451          * in init_config().
 452          */
 453         .max_rx_pkt_len = 0,
 454 };
 455
 456 struct rte_eth_txmode tx_mode = {
 457         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 458 };
 459
 460 struct rte_fdir_conf fdir_conf = {
 461         .mode = RTE_FDIR_MODE_NONE,
 462         .pballoc = RTE_FDIR_PBALLOC_64K,
 463         .status = RTE_FDIR_REPORT_STATUS,
 464         .mask = {
 465                 .vlan_tci_mask = 0xFFEF,
 466                 .ipv4_mask     = {
 467                         .src_ip = 0xFFFFFFFF,
 468                         .dst_ip = 0xFFFFFFFF,
 469                 },
 470                 .ipv6_mask     = {
 471                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 472                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 473                 },
 474                 .src_port_mask = 0xFFFF,
 475                 .dst_port_mask = 0xFFFF,
 476                 .mac_addr_byte_mask = 0xFF,
 477                 .tunnel_type_mask = 1,
 478                 .tunnel_id_mask = 0xFFFFFFFF,
 479         },
 480         .drop_queue = 127,
 481 };
 482
 483 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 484
 485 /*
 486  * Display zero values by default for xstats
 487  */
 488 uint8_t xstats_hide_zero;
 489
 490 /*
 491  * Measure of CPU cycles disabled by default
 492  */
 493 uint8_t record_core_cycles;
 494
 495 /*
 496  * Display of RX and TX bursts disabled by default
 497  */
 498 uint8_t record_burst_stats;
 499
 500 unsigned int num_sockets = 0;
 501 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 502
 503 #ifdef RTE_LIB_BITRATESTATS
 504 /* Bitrate statistics */
 505 struct rte_stats_bitrates *bitrate_data;
 506 lcoreid_t bitrate_lcore_id;
 507 uint8_t bitrate_enabled;
 508 #endif
 509
 510 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 511 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 512
 513 /*
 514  * hexadecimal bitmask of RX mq mode can be enabled.
 515  */
 516 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 517
 518 /*
 519  * Used to set forced link speed
 520  */
 521 uint32_t eth_link_speed;
 522
 523 /* Forward function declarations */
 524 static void setup_attached_port(portid_t pi);
 525 static void check_all_ports_link_status(uint32_t port_mask);
 526 static int eth_event_callback(portid_t port_id,
 527                               enum rte_eth_event_type type,
 528                               void *param, void *ret_param);
 529 static void dev_event_callback(const char *device_name,
 530                                 enum rte_dev_event_type type,
 531                                 void *param);
 532
 533 /*
 534  * Check if all the ports are started.
 535  * If yes, return positive value. If not, return zero.
 536  */
 537 static int all_ports_started(void);
 538
 539 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 540 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 541
 542 /* Holds the registered mbuf dynamic flags names. */
 543 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 544
 545 /*
 546  * Helper function to check if socket is already discovered.
 547  * If yes, return positive value. If not, return zero.
 548  */
 549 int
 550 new_socket_id(unsigned int socket_id)
 551 {
 552         unsigned int i;
 553
 554         for (i = 0; i < num_sockets; i++) {
 555                 if (socket_ids[i] == socket_id)
 556                         return 0;
 557         }
 558         return 1;
 559 }
 560
 561 /*
 562  * Setup default configuration.
 563  */
 564 static void
 565 set_default_fwd_lcores_config(void)
 566 {
 567         unsigned int i;
 568         unsigned int nb_lc;
 569         unsigned int sock_num;
 570
 571         nb_lc = 0;
 572         for (i = 0; i < RTE_MAX_LCORE; i++) {
 573                 if (!rte_lcore_is_enabled(i))
 574                         continue;
 575                 sock_num = rte_lcore_to_socket_id(i);
 576                 if (new_socket_id(sock_num)) {
 577                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 578                                 rte_exit(EXIT_FAILURE,
 579                                          "Total sockets greater than %u\n",
 580                                          RTE_MAX_NUMA_NODES);
 581                         }
 582                         socket_ids[num_sockets++] = sock_num;
 583                 }
 584                 if (i == rte_get_main_lcore())
 585                         continue;
 586                 fwd_lcores_cpuids[nb_lc++] = i;
 587         }
 588         nb_lcores = (lcoreid_t) nb_lc;
 589         nb_cfg_lcores = nb_lcores;
 590         nb_fwd_lcores = 1;
 591 }
 592
 593 static void
 594 set_def_peer_eth_addrs(void)
 595 {
 596         portid_t i;
 597
 598         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 599                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 600                 peer_eth_addrs[i].addr_bytes[5] = i;
 601         }
 602 }
 603
 604 static void
 605 set_default_fwd_ports_config(void)
 606 {
 607         portid_t pt_id;
 608         int i = 0;
 609
 610         RTE_ETH_FOREACH_DEV(pt_id) {
 611                 fwd_ports_ids[i++] = pt_id;
 612
 613                 /* Update sockets info according to the attached device */
 614                 int socket_id = rte_eth_dev_socket_id(pt_id);
 615                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 616                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 617                                 rte_exit(EXIT_FAILURE,
 618                                          "Total sockets greater than %u\n",
 619                                          RTE_MAX_NUMA_NODES);
 620                         }
 621                         socket_ids[num_sockets++] = socket_id;
 622                 }
 623         }
 624
 625         nb_cfg_ports = nb_ports;
 626         nb_fwd_ports = nb_ports;
 627 }
 628
 629 void
 630 set_def_fwd_config(void)
 631 {
 632         set_default_fwd_lcores_config();
 633         set_def_peer_eth_addrs();
 634         set_default_fwd_ports_config();
 635 }
 636
 637 #ifndef RTE_EXEC_ENV_WINDOWS
 638 /* extremely pessimistic estimation of memory required to create a mempool */
 639 static int
 640 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 641 {
 642         unsigned int n_pages, mbuf_per_pg, leftover;
 643         uint64_t total_mem, mbuf_mem, obj_sz;
 644
 645         /* there is no good way to predict how much space the mempool will
 646          * occupy because it will allocate chunks on the fly, and some of those
 647          * will come from default DPDK memory while some will come from our
 648          * external memory, so just assume 128MB will be enough for everyone.
 649          */
 650         uint64_t hdr_mem = 128 << 20;
 651
 652         /* account for possible non-contiguousness */
 653         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 654         if (obj_sz > pgsz) {
 655                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 656                 return -1;
 657         }
 658
 659         mbuf_per_pg = pgsz / obj_sz;
 660         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 661         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 662
 663         mbuf_mem = n_pages * pgsz;
 664
 665         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 666
 667         if (total_mem > SIZE_MAX) {
 668                 TESTPMD_LOG(ERR, "Memory size too big\n");
 669                 return -1;
 670         }
 671         *out = (size_t)total_mem;
 672
 673         return 0;
 674 }
 675
 676 static int
 677 pagesz_flags(uint64_t page_sz)
 678 {
 679         /* as per mmap() manpage, all page sizes are log2 of page size
 680          * shifted by MAP_HUGE_SHIFT
 681          */
 682         int log2 = rte_log2_u64(page_sz);
 683
 684         return (log2 << HUGE_SHIFT);
 685 }
 686
 687 static void *
 688 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 689 {
 690         void *addr;
 691         int flags;
 692
 693         /* allocate anonymous hugepages */
 694         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 695         if (huge)
 696                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 697
 698         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 699         if (addr == MAP_FAILED)
 700                 return NULL;
 701
 702         return addr;
 703 }
 704
 705 struct extmem_param {
 706         void *addr;
 707         size_t len;
 708         size_t pgsz;
 709         rte_iova_t *iova_table;
 710         unsigned int iova_table_len;
 711 };
 712
 713 static int
 714 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 715                 bool huge)
 716 {
 717         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 718                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 719         unsigned int cur_page, n_pages, pgsz_idx;
 720         size_t mem_sz, cur_pgsz;
 721         rte_iova_t *iovas = NULL;
 722         void *addr;
 723         int ret;
 724
 725         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 726                 /* skip anything that is too big */
 727                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 728                         continue;
 729
 730                 cur_pgsz = pgsizes[pgsz_idx];
 731
 732                 /* if we were told not to allocate hugepages, override */
 733                 if (!huge)
 734                         cur_pgsz = sysconf(_SC_PAGESIZE);
 735
 736                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 737                 if (ret < 0) {
 738                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 739                         return -1;
 740                 }
 741
 742                 /* allocate our memory */
 743                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 744
 745                 /* if we couldn't allocate memory with a specified page size,
 746                  * that doesn't mean we can't do it with other page sizes, so
 747                  * try another one.
 748                  */
 749                 if (addr == NULL)
 750                         continue;
 751
 752                 /* store IOVA addresses for every page in this memory area */
 753                 n_pages = mem_sz / cur_pgsz;
 754
 755                 iovas = malloc(sizeof(*iovas) * n_pages);
 756
 757                 if (iovas == NULL) {
 758                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 759                         goto fail;
 760                 }
 761                 /* lock memory if it's not huge pages */
 762                 if (!huge)
 763                         mlock(addr, mem_sz);
 764
 765                 /* populate IOVA addresses */
 766                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 767                         rte_iova_t iova;
 768                         size_t offset;
 769                         void *cur;
 770
 771                         offset = cur_pgsz * cur_page;
 772                         cur = RTE_PTR_ADD(addr, offset);
 773
 774                         /* touch the page before getting its IOVA */
 775                         *(volatile char *)cur = 0;
 776
 777                         iova = rte_mem_virt2iova(cur);
 778
 779                         iovas[cur_page] = iova;
 780                 }
 781
 782                 break;
 783         }
 784         /* if we couldn't allocate anything */
 785         if (iovas == NULL)
 786                 return -1;
 787
 788         param->addr = addr;
 789         param->len = mem_sz;
 790         param->pgsz = cur_pgsz;
 791         param->iova_table = iovas;
 792         param->iova_table_len = n_pages;
 793
 794         return 0;
 795 fail:
 796         if (iovas)
 797                 free(iovas);
 798         if (addr)
 799                 munmap(addr, mem_sz);
 800
 801         return -1;
 802 }
 803
 804 static int
 805 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 806 {
 807         struct extmem_param param;
 808         int socket_id, ret;
 809
 810         memset(&param, 0, sizeof(param));
 811
 812         /* check if our heap exists */
 813         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 814         if (socket_id < 0) {
 815                 /* create our heap */
 816                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 817                 if (ret < 0) {
 818                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 819                         return -1;
 820                 }
 821         }
 822
 823         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 824         if (ret < 0) {
 825                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 826                 return -1;
 827         }
 828
 829         /* we now have a valid memory area, so add it to heap */
 830         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 831                         param.addr, param.len, param.iova_table,
 832                         param.iova_table_len, param.pgsz);
 833
 834         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 835
 836         /* not needed any more */
 837         free(param.iova_table);
 838
 839         if (ret < 0) {
 840                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 841                 munmap(param.addr, param.len);
 842                 return -1;
 843         }
 844
 845         /* success */
 846
 847         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 848                         param.len >> 20);
 849
 850         return 0;
 851 }
 852 static void
 853 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 854              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 855 {
 856         uint16_t pid = 0;
 857         int ret;
 858
 859         RTE_ETH_FOREACH_DEV(pid) {
 860                 struct rte_eth_dev *dev =
 861                         &rte_eth_devices[pid];
 862
 863                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 864                                         memhdr->len);
 865                 if (ret) {
 866                         TESTPMD_LOG(DEBUG,
 867                                     "unable to DMA unmap addr 0x%p "
 868                                     "for device %s\n",
 869                                     memhdr->addr, dev->data->name);
 870                 }
 871         }
 872         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 873         if (ret) {
 874                 TESTPMD_LOG(DEBUG,
 875                             "unable to un-register addr 0x%p\n", memhdr->addr);
 876         }
 877 }
 878
 879 static void
 880 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 881            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 882 {
 883         uint16_t pid = 0;
 884         size_t page_size = sysconf(_SC_PAGESIZE);
 885         int ret;
 886
 887         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 888                                   page_size);
 889         if (ret) {
 890                 TESTPMD_LOG(DEBUG,
 891                             "unable to register addr 0x%p\n", memhdr->addr);
 892                 return;
 893         }
 894         RTE_ETH_FOREACH_DEV(pid) {
 895                 struct rte_eth_dev *dev =
 896                         &rte_eth_devices[pid];
 897
 898                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 899                                       memhdr->len);
 900                 if (ret) {
 901                         TESTPMD_LOG(DEBUG,
 902                                     "unable to DMA map addr 0x%p "
 903                                     "for device %s\n",
 904                                     memhdr->addr, dev->data->name);
 905                 }
 906         }
 907 }
 908 #endif
 909
 910 static unsigned int
 911 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 912             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 913 {
 914         struct rte_pktmbuf_extmem *xmem;
 915         unsigned int ext_num, zone_num, elt_num;
 916         uint16_t elt_size;
 917
 918         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 919         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 920         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 921
 922         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 923         if (xmem == NULL) {
 924                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 925                                  "external buffer descriptors\n");
 926                 *ext_mem = NULL;
 927                 return 0;
 928         }
 929         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 930                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 931                 const struct rte_memzone *mz;
 932                 char mz_name[RTE_MEMZONE_NAMESIZE];
 933                 int ret;
 934
 935                 ret = snprintf(mz_name, sizeof(mz_name),
 936                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 937                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 938                         errno = ENAMETOOLONG;
 939                         ext_num = 0;
 940                         break;
 941                 }
 942                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 943                                                  socket_id,
 944                                                  RTE_MEMZONE_IOVA_CONTIG |
 945                                                  RTE_MEMZONE_1GB |
 946                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 947                                                  EXTBUF_ZONE_SIZE);
 948                 if (mz == NULL) {
 949                         /*
 950                          * The caller exits on external buffer creation
 951                          * error, so there is no need to free memzones.
 952                          */
 953                         errno = ENOMEM;
 954                         ext_num = 0;
 955                         break;
 956                 }
 957                 xseg->buf_ptr = mz->addr;
 958                 xseg->buf_iova = mz->iova;
 959                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 960                 xseg->elt_size = elt_size;
 961         }
 962         if (ext_num == 0 && xmem != NULL) {
 963                 free(xmem);
 964                 xmem = NULL;
 965         }
 966         *ext_mem = xmem;
 967         return ext_num;
 968 }
 969
 970 /*
 971  * Configuration initialisation done once at init time.
 972  */
 973 static struct rte_mempool *
 974 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 975                  unsigned int socket_id, uint16_t size_idx)
 976 {
 977         char pool_name[RTE_MEMPOOL_NAMESIZE];
 978         struct rte_mempool *rte_mp = NULL;
 979 #ifndef RTE_EXEC_ENV_WINDOWS
 980         uint32_t mb_size;
 981
 982         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 983 #endif
 984         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 985
 986         TESTPMD_LOG(INFO,
 987                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 988                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 989
 990         switch (mp_alloc_type) {
 991         case MP_ALLOC_NATIVE:
 992                 {
 993                         /* wrapper to rte_mempool_create() */
 994                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 995                                         rte_mbuf_best_mempool_ops());
 996                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 997                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 998                         break;
 999                 }
1000 #ifndef RTE_EXEC_ENV_WINDOWS
1001         case MP_ALLOC_ANON:
1002                 {
1003                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1004                                 mb_size, (unsigned int) mb_mempool_cache,
1005                                 sizeof(struct rte_pktmbuf_pool_private),
1006                                 socket_id, mempool_flags);
1007                         if (rte_mp == NULL)
1008                                 goto err;
1009
1010                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1011                                 rte_mempool_free(rte_mp);
1012                                 rte_mp = NULL;
1013                                 goto err;
1014                         }
1015                         rte_pktmbuf_pool_init(rte_mp, NULL);
1016                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1017                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1018                         break;
1019                 }
1020         case MP_ALLOC_XMEM:
1021         case MP_ALLOC_XMEM_HUGE:
1022                 {
1023                         int heap_socket;
1024                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1025
1026                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1027                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1028
1029                         heap_socket =
1030                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1031                         if (heap_socket < 0)
1032                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1033
1034                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1035                                         rte_mbuf_best_mempool_ops());
1036                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1037                                         mb_mempool_cache, 0, mbuf_seg_size,
1038                                         heap_socket);
1039                         break;
1040                 }
1041 #endif
1042         case MP_ALLOC_XBUF:
1043                 {
1044                         struct rte_pktmbuf_extmem *ext_mem;
1045                         unsigned int ext_num;
1046
1047                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1048                                                socket_id, pool_name, &ext_mem);
1049                         if (ext_num == 0)
1050                                 rte_exit(EXIT_FAILURE,
1051                                          "Can't create pinned data buffers\n");
1052
1053                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1054                                         rte_mbuf_best_mempool_ops());
1055                         rte_mp = rte_pktmbuf_pool_create_extbuf
1056                                         (pool_name, nb_mbuf, mb_mempool_cache,
1057                                          0, mbuf_seg_size, socket_id,
1058                                          ext_mem, ext_num);
1059                         free(ext_mem);
1060                         break;
1061                 }
1062         default:
1063                 {
1064                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1065                 }
1066         }
1067
1068 #ifndef RTE_EXEC_ENV_WINDOWS
1069 err:
1070 #endif
1071         if (rte_mp == NULL) {
1072                 rte_exit(EXIT_FAILURE,
1073                         "Creation of mbuf pool for socket %u failed: %s\n",
1074                         socket_id, rte_strerror(rte_errno));
1075         } else if (verbose_level > 0) {
1076                 rte_mempool_dump(stdout, rte_mp);
1077         }
1078         return rte_mp;
1079 }
1080
1081 /*
1082  * Check given socket id is valid or not with NUMA mode,
1083  * if valid, return 0, else return -1
1084  */
1085 static int
1086 check_socket_id(const unsigned int socket_id)
1087 {
1088         static int warning_once = 0;
1089
1090         if (new_socket_id(socket_id)) {
1091                 if (!warning_once && numa_support)
1092                         printf("Warning: NUMA should be configured manually by"
1093                                " using --port-numa-config and"
1094                                " --ring-numa-config parameters along with"
1095                                " --numa.\n");
1096                 warning_once = 1;
1097                 return -1;
1098         }
1099         return 0;
1100 }
1101
1102 /*
1103  * Get the allowed maximum number of RX queues.
1104  * *pid return the port id which has minimal value of
1105  * max_rx_queues in all ports.
1106  */
1107 queueid_t
1108 get_allowed_max_nb_rxq(portid_t *pid)
1109 {
1110         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1111         bool max_rxq_valid = false;
1112         portid_t pi;
1113         struct rte_eth_dev_info dev_info;
1114
1115         RTE_ETH_FOREACH_DEV(pi) {
1116                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1117                         continue;
1118
1119                 max_rxq_valid = true;
1120                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1121                         allowed_max_rxq = dev_info.max_rx_queues;
1122                         *pid = pi;
1123                 }
1124         }
1125         return max_rxq_valid ? allowed_max_rxq : 0;
1126 }
1127
1128 /*
1129  * Check input rxq is valid or not.
1130  * If input rxq is not greater than any of maximum number
1131  * of RX queues of all ports, it is valid.
1132  * if valid, return 0, else return -1
1133  */
1134 int
1135 check_nb_rxq(queueid_t rxq)
1136 {
1137         queueid_t allowed_max_rxq;
1138         portid_t pid = 0;
1139
1140         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1141         if (rxq > allowed_max_rxq) {
1142                 printf("Fail: input rxq (%u) can't be greater "
1143                        "than max_rx_queues (%u) of port %u\n",
1144                        rxq,
1145                        allowed_max_rxq,
1146                        pid);
1147                 return -1;
1148         }
1149         return 0;
1150 }
1151
1152 /*
1153  * Get the allowed maximum number of TX queues.
1154  * *pid return the port id which has minimal value of
1155  * max_tx_queues in all ports.
1156  */
1157 queueid_t
1158 get_allowed_max_nb_txq(portid_t *pid)
1159 {
1160         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1161         bool max_txq_valid = false;
1162         portid_t pi;
1163         struct rte_eth_dev_info dev_info;
1164
1165         RTE_ETH_FOREACH_DEV(pi) {
1166                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1167                         continue;
1168
1169                 max_txq_valid = true;
1170                 if (dev_info.max_tx_queues < allowed_max_txq) {
1171                         allowed_max_txq = dev_info.max_tx_queues;
1172                         *pid = pi;
1173                 }
1174         }
1175         return max_txq_valid ? allowed_max_txq : 0;
1176 }
1177
1178 /*
1179  * Check input txq is valid or not.
1180  * If input txq is not greater than any of maximum number
1181  * of TX queues of all ports, it is valid.
1182  * if valid, return 0, else return -1
1183  */
1184 int
1185 check_nb_txq(queueid_t txq)
1186 {
1187         queueid_t allowed_max_txq;
1188         portid_t pid = 0;
1189
1190         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1191         if (txq > allowed_max_txq) {
1192                 printf("Fail: input txq (%u) can't be greater "
1193                        "than max_tx_queues (%u) of port %u\n",
1194                        txq,
1195                        allowed_max_txq,
1196                        pid);
1197                 return -1;
1198         }
1199         return 0;
1200 }
1201
1202 /*
1203  * Get the allowed maximum number of RXDs of every rx queue.
1204  * *pid return the port id which has minimal value of
1205  * max_rxd in all queues of all ports.
1206  */
1207 static uint16_t
1208 get_allowed_max_nb_rxd(portid_t *pid)
1209 {
1210         uint16_t allowed_max_rxd = UINT16_MAX;
1211         portid_t pi;
1212         struct rte_eth_dev_info dev_info;
1213
1214         RTE_ETH_FOREACH_DEV(pi) {
1215                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1216                         continue;
1217
1218                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1219                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1220                         *pid = pi;
1221                 }
1222         }
1223         return allowed_max_rxd;
1224 }
1225
1226 /*
1227  * Get the allowed minimal number of RXDs of every rx queue.
1228  * *pid return the port id which has minimal value of
1229  * min_rxd in all queues of all ports.
1230  */
1231 static uint16_t
1232 get_allowed_min_nb_rxd(portid_t *pid)
1233 {
1234         uint16_t allowed_min_rxd = 0;
1235         portid_t pi;
1236         struct rte_eth_dev_info dev_info;
1237
1238         RTE_ETH_FOREACH_DEV(pi) {
1239                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1240                         continue;
1241
1242                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1243                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1244                         *pid = pi;
1245                 }
1246         }
1247
1248         return allowed_min_rxd;
1249 }
1250
1251 /*
1252  * Check input rxd is valid or not.
1253  * If input rxd is not greater than any of maximum number
1254  * of RXDs of every Rx queues and is not less than any of
1255  * minimal number of RXDs of every Rx queues, it is valid.
1256  * if valid, return 0, else return -1
1257  */
1258 int
1259 check_nb_rxd(queueid_t rxd)
1260 {
1261         uint16_t allowed_max_rxd;
1262         uint16_t allowed_min_rxd;
1263         portid_t pid = 0;
1264
1265         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1266         if (rxd > allowed_max_rxd) {
1267                 printf("Fail: input rxd (%u) can't be greater "
1268                        "than max_rxds (%u) of port %u\n",
1269                        rxd,
1270                        allowed_max_rxd,
1271                        pid);
1272                 return -1;
1273         }
1274
1275         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1276         if (rxd < allowed_min_rxd) {
1277                 printf("Fail: input rxd (%u) can't be less "
1278                        "than min_rxds (%u) of port %u\n",
1279                        rxd,
1280                        allowed_min_rxd,
1281                        pid);
1282                 return -1;
1283         }
1284
1285         return 0;
1286 }
1287
1288 /*
1289  * Get the allowed maximum number of TXDs of every rx queues.
1290  * *pid return the port id which has minimal value of
1291  * max_txd in every tx queue.
1292  */
1293 static uint16_t
1294 get_allowed_max_nb_txd(portid_t *pid)
1295 {
1296         uint16_t allowed_max_txd = UINT16_MAX;
1297         portid_t pi;
1298         struct rte_eth_dev_info dev_info;
1299
1300         RTE_ETH_FOREACH_DEV(pi) {
1301                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1302                         continue;
1303
1304                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1305                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1306                         *pid = pi;
1307                 }
1308         }
1309         return allowed_max_txd;
1310 }
1311
1312 /*
1313  * Get the allowed maximum number of TXDs of every tx queues.
1314  * *pid return the port id which has minimal value of
1315  * min_txd in every tx queue.
1316  */
1317 static uint16_t
1318 get_allowed_min_nb_txd(portid_t *pid)
1319 {
1320         uint16_t allowed_min_txd = 0;
1321         portid_t pi;
1322         struct rte_eth_dev_info dev_info;
1323
1324         RTE_ETH_FOREACH_DEV(pi) {
1325                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1326                         continue;
1327
1328                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1329                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1330                         *pid = pi;
1331                 }
1332         }
1333
1334         return allowed_min_txd;
1335 }
1336
1337 /*
1338  * Check input txd is valid or not.
1339  * If input txd is not greater than any of maximum number
1340  * of TXDs of every Rx queues, it is valid.
1341  * if valid, return 0, else return -1
1342  */
1343 int
1344 check_nb_txd(queueid_t txd)
1345 {
1346         uint16_t allowed_max_txd;
1347         uint16_t allowed_min_txd;
1348         portid_t pid = 0;
1349
1350         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1351         if (txd > allowed_max_txd) {
1352                 printf("Fail: input txd (%u) can't be greater "
1353                        "than max_txds (%u) of port %u\n",
1354                        txd,
1355                        allowed_max_txd,
1356                        pid);
1357                 return -1;
1358         }
1359
1360         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1361         if (txd < allowed_min_txd) {
1362                 printf("Fail: input txd (%u) can't be less "
1363                        "than min_txds (%u) of port %u\n",
1364                        txd,
1365                        allowed_min_txd,
1366                        pid);
1367                 return -1;
1368         }
1369         return 0;
1370 }
1371
1372
1373 /*
1374  * Get the allowed maximum number of hairpin queues.
1375  * *pid return the port id which has minimal value of
1376  * max_hairpin_queues in all ports.
1377  */
1378 queueid_t
1379 get_allowed_max_nb_hairpinq(portid_t *pid)
1380 {
1381         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1382         portid_t pi;
1383         struct rte_eth_hairpin_cap cap;
1384
1385         RTE_ETH_FOREACH_DEV(pi) {
1386                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1387                         *pid = pi;
1388                         return 0;
1389                 }
1390                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1391                         allowed_max_hairpinq = cap.max_nb_queues;
1392                         *pid = pi;
1393                 }
1394         }
1395         return allowed_max_hairpinq;
1396 }
1397
1398 /*
1399  * Check input hairpin is valid or not.
1400  * If input hairpin is not greater than any of maximum number
1401  * of hairpin queues of all ports, it is valid.
1402  * if valid, return 0, else return -1
1403  */
1404 int
1405 check_nb_hairpinq(queueid_t hairpinq)
1406 {
1407         queueid_t allowed_max_hairpinq;
1408         portid_t pid = 0;
1409
1410         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1411         if (hairpinq > allowed_max_hairpinq) {
1412                 printf("Fail: input hairpin (%u) can't be greater "
1413                        "than max_hairpin_queues (%u) of port %u\n",
1414                        hairpinq, allowed_max_hairpinq, pid);
1415                 return -1;
1416         }
1417         return 0;
1418 }
1419
1420 static void
1421 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1422 {
1423         struct rte_port *port = &ports[pid];
1424         uint16_t data_size;
1425         int ret;
1426         int i;
1427
1428         port->dev_conf.txmode = tx_mode;
1429         port->dev_conf.rxmode = rx_mode;
1430
1431         ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1432         if (ret != 0)
1433                 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1434
1435         ret = update_jumbo_frame_offload(pid);
1436         if (ret != 0)
1437                 printf("Updating jumbo frame offload failed for port %u\n",
1438                         pid);
1439
1440         if (!(port->dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1441                 port->dev_conf.txmode.offloads &=
1442                         ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1443
1444         /* Apply Rx offloads configuration */
1445         for (i = 0; i < port->dev_info.max_rx_queues; i++)
1446                 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads;
1447         /* Apply Tx offloads configuration */
1448         for (i = 0; i < port->dev_info.max_tx_queues; i++)
1449                 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads;
1450
1451         if (eth_link_speed)
1452                 port->dev_conf.link_speeds = eth_link_speed;
1453
1454         /* set flag to initialize port/queue */
1455         port->need_reconfig = 1;
1456         port->need_reconfig_queues = 1;
1457         port->socket_id = socket_id;
1458         port->tx_metadata = 0;
1459
1460         /*
1461          * Check for maximum number of segments per MTU.
1462          * Accordingly update the mbuf data size.
1463          */
1464         if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1465             port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1466                 data_size = rx_mode.max_rx_pkt_len /
1467                         port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1468
1469                 if ((data_size + RTE_PKTMBUF_HEADROOM) > mbuf_data_size[0]) {
1470                         mbuf_data_size[0] = data_size + RTE_PKTMBUF_HEADROOM;
1471                         TESTPMD_LOG(WARNING,
1472                                     "Configured mbuf size of the first segment %hu\n",
1473                                     mbuf_data_size[0]);
1474                 }
1475         }
1476 }
1477
1478 static void
1479 init_config(void)
1480 {
1481         portid_t pid;
1482         struct rte_mempool *mbp;
1483         unsigned int nb_mbuf_per_pool;
1484         lcoreid_t  lc_id;
1485         struct rte_gro_param gro_param;
1486         uint32_t gso_types;
1487
1488         /* Configuration of logical cores. */
1489         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1490                                 sizeof(struct fwd_lcore *) * nb_lcores,
1491                                 RTE_CACHE_LINE_SIZE);
1492         if (fwd_lcores == NULL) {
1493                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1494                                                         "failed\n", nb_lcores);
1495         }
1496         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1497                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1498                                                sizeof(struct fwd_lcore),
1499                                                RTE_CACHE_LINE_SIZE);
1500                 if (fwd_lcores[lc_id] == NULL) {
1501                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1502                                                                 "failed\n");
1503                 }
1504                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1505         }
1506
1507         RTE_ETH_FOREACH_DEV(pid) {
1508                 uint32_t socket_id;
1509
1510                 if (numa_support) {
1511                         socket_id = port_numa[pid];
1512                         if (port_numa[pid] == NUMA_NO_CONFIG) {
1513                                 socket_id = rte_eth_dev_socket_id(pid);
1514
1515                                 /*
1516                                  * if socket_id is invalid,
1517                                  * set to the first available socket.
1518                                  */
1519                                 if (check_socket_id(socket_id) < 0)
1520                                         socket_id = socket_ids[0];
1521                         }
1522                 } else {
1523                         socket_id = (socket_num == UMA_NO_CONFIG) ?
1524                                     0 : socket_num;
1525                 }
1526                 /* Apply default TxRx configuration for all ports */
1527                 init_config_port_offloads(pid, socket_id);
1528         }
1529         /*
1530          * Create pools of mbuf.
1531          * If NUMA support is disabled, create a single pool of mbuf in
1532          * socket 0 memory by default.
1533          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1534          *
1535          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1536          * nb_txd can be configured at run time.
1537          */
1538         if (param_total_num_mbufs)
1539                 nb_mbuf_per_pool = param_total_num_mbufs;
1540         else {
1541                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1542                         (nb_lcores * mb_mempool_cache) +
1543                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1544                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1545         }
1546
1547         if (numa_support) {
1548                 uint8_t i, j;
1549
1550                 for (i = 0; i < num_sockets; i++)
1551                         for (j = 0; j < mbuf_data_size_n; j++)
1552                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1553                                         mbuf_pool_create(mbuf_data_size[j],
1554                                                           nb_mbuf_per_pool,
1555                                                           socket_ids[i], j);
1556         } else {
1557                 uint8_t i;
1558
1559                 for (i = 0; i < mbuf_data_size_n; i++)
1560                         mempools[i] = mbuf_pool_create
1561                                         (mbuf_data_size[i],
1562                                          nb_mbuf_per_pool,
1563                                          socket_num == UMA_NO_CONFIG ?
1564                                          0 : socket_num, i);
1565         }
1566
1567         init_port_config();
1568
1569         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1570                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1571         /*
1572          * Records which Mbuf pool to use by each logical core, if needed.
1573          */
1574         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1575                 mbp = mbuf_pool_find(
1576                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1577
1578                 if (mbp == NULL)
1579                         mbp = mbuf_pool_find(0, 0);
1580                 fwd_lcores[lc_id]->mbp = mbp;
1581                 /* initialize GSO context */
1582                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1583                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1584                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1585                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1586                         RTE_ETHER_CRC_LEN;
1587                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1588         }
1589
1590         fwd_config_setup();
1591
1592         /* create a gro context for each lcore */
1593         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1594         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1595         gro_param.max_item_per_flow = MAX_PKT_BURST;
1596         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1597                 gro_param.socket_id = rte_lcore_to_socket_id(
1598                                 fwd_lcores_cpuids[lc_id]);
1599                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1600                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1601                         rte_exit(EXIT_FAILURE,
1602                                         "rte_gro_ctx_create() failed\n");
1603                 }
1604         }
1605 }
1606
1607
1608 void
1609 reconfig(portid_t new_port_id, unsigned socket_id)
1610 {
1611         /* Reconfiguration of Ethernet ports. */
1612         init_config_port_offloads(new_port_id, socket_id);
1613         init_port_config();
1614 }
1615
1616
1617 int
1618 init_fwd_streams(void)
1619 {
1620         portid_t pid;
1621         struct rte_port *port;
1622         streamid_t sm_id, nb_fwd_streams_new;
1623         queueid_t q;
1624
1625         /* set socket id according to numa or not */
1626         RTE_ETH_FOREACH_DEV(pid) {
1627                 port = &ports[pid];
1628                 if (nb_rxq > port->dev_info.max_rx_queues) {
1629                         printf("Fail: nb_rxq(%d) is greater than "
1630                                 "max_rx_queues(%d)\n", nb_rxq,
1631                                 port->dev_info.max_rx_queues);
1632                         return -1;
1633                 }
1634                 if (nb_txq > port->dev_info.max_tx_queues) {
1635                         printf("Fail: nb_txq(%d) is greater than "
1636                                 "max_tx_queues(%d)\n", nb_txq,
1637                                 port->dev_info.max_tx_queues);
1638                         return -1;
1639                 }
1640                 if (numa_support) {
1641                         if (port_numa[pid] != NUMA_NO_CONFIG)
1642                                 port->socket_id = port_numa[pid];
1643                         else {
1644                                 port->socket_id = rte_eth_dev_socket_id(pid);
1645
1646                                 /*
1647                                  * if socket_id is invalid,
1648                                  * set to the first available socket.
1649                                  */
1650                                 if (check_socket_id(port->socket_id) < 0)
1651                                         port->socket_id = socket_ids[0];
1652                         }
1653                 }
1654                 else {
1655                         if (socket_num == UMA_NO_CONFIG)
1656                                 port->socket_id = 0;
1657                         else
1658                                 port->socket_id = socket_num;
1659                 }
1660         }
1661
1662         q = RTE_MAX(nb_rxq, nb_txq);
1663         if (q == 0) {
1664                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1665                 return -1;
1666         }
1667         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1668         if (nb_fwd_streams_new == nb_fwd_streams)
1669                 return 0;
1670         /* clear the old */
1671         if (fwd_streams != NULL) {
1672                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1673                         if (fwd_streams[sm_id] == NULL)
1674                                 continue;
1675                         rte_free(fwd_streams[sm_id]);
1676                         fwd_streams[sm_id] = NULL;
1677                 }
1678                 rte_free(fwd_streams);
1679                 fwd_streams = NULL;
1680         }
1681
1682         /* init new */
1683         nb_fwd_streams = nb_fwd_streams_new;
1684         if (nb_fwd_streams) {
1685                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1686                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1687                         RTE_CACHE_LINE_SIZE);
1688                 if (fwd_streams == NULL)
1689                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1690                                  " (struct fwd_stream *)) failed\n",
1691                                  nb_fwd_streams);
1692
1693                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1694                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1695                                 " struct fwd_stream", sizeof(struct fwd_stream),
1696                                 RTE_CACHE_LINE_SIZE);
1697                         if (fwd_streams[sm_id] == NULL)
1698                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1699                                          "(struct fwd_stream) failed\n");
1700                 }
1701         }
1702
1703         return 0;
1704 }
1705
1706 static void
1707 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1708 {
1709         uint64_t total_burst, sburst;
1710         uint64_t nb_burst;
1711         uint64_t burst_stats[4];
1712         uint16_t pktnb_stats[4];
1713         uint16_t nb_pkt;
1714         int burst_percent[4], sburstp;
1715         int i;
1716
1717         /*
1718          * First compute the total number of packet bursts and the
1719          * two highest numbers of bursts of the same number of packets.
1720          */
1721         memset(&burst_stats, 0x0, sizeof(burst_stats));
1722         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1723
1724         /* Show stats for 0 burst size always */
1725         total_burst = pbs->pkt_burst_spread[0];
1726         burst_stats[0] = pbs->pkt_burst_spread[0];
1727         pktnb_stats[0] = 0;
1728
1729         /* Find the next 2 burst sizes with highest occurrences. */
1730         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1731                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1732
1733                 if (nb_burst == 0)
1734                         continue;
1735
1736                 total_burst += nb_burst;
1737
1738                 if (nb_burst > burst_stats[1]) {
1739                         burst_stats[2] = burst_stats[1];
1740                         pktnb_stats[2] = pktnb_stats[1];
1741                         burst_stats[1] = nb_burst;
1742                         pktnb_stats[1] = nb_pkt;
1743                 } else if (nb_burst > burst_stats[2]) {
1744                         burst_stats[2] = nb_burst;
1745                         pktnb_stats[2] = nb_pkt;
1746                 }
1747         }
1748         if (total_burst == 0)
1749                 return;
1750
1751         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1752         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1753                 if (i == 3) {
1754                         printf("%d%% of other]\n", 100 - sburstp);
1755                         return;
1756                 }
1757
1758                 sburst += burst_stats[i];
1759                 if (sburst == total_burst) {
1760                         printf("%d%% of %d pkts]\n",
1761                                 100 - sburstp, (int) pktnb_stats[i]);
1762                         return;
1763                 }
1764
1765                 burst_percent[i] =
1766                         (double)burst_stats[i] / total_burst * 100;
1767                 printf("%d%% of %d pkts + ",
1768                         burst_percent[i], (int) pktnb_stats[i]);
1769                 sburstp += burst_percent[i];
1770         }
1771 }
1772
1773 static void
1774 fwd_stream_stats_display(streamid_t stream_id)
1775 {
1776         struct fwd_stream *fs;
1777         static const char *fwd_top_stats_border = "-------";
1778
1779         fs = fwd_streams[stream_id];
1780         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1781             (fs->fwd_dropped == 0))
1782                 return;
1783         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1784                "TX Port=%2d/Queue=%2d %s\n",
1785                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1786                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1787         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1788                " TX-dropped: %-14"PRIu64,
1789                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1790
1791         /* if checksum mode */
1792         if (cur_fwd_eng == &csum_fwd_engine) {
1793                 printf("  RX- bad IP checksum: %-14"PRIu64
1794                        "  Rx- bad L4 checksum: %-14"PRIu64
1795                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1796                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1797                         fs->rx_bad_outer_l4_csum);
1798                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1799                         fs->rx_bad_outer_ip_csum);
1800         } else {
1801                 printf("\n");
1802         }
1803
1804         if (record_burst_stats) {
1805                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1806                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1807         }
1808 }
1809
1810 void
1811 fwd_stats_display(void)
1812 {
1813         static const char *fwd_stats_border = "----------------------";
1814         static const char *acc_stats_border = "+++++++++++++++";
1815         struct {
1816                 struct fwd_stream *rx_stream;
1817                 struct fwd_stream *tx_stream;
1818                 uint64_t tx_dropped;
1819                 uint64_t rx_bad_ip_csum;
1820                 uint64_t rx_bad_l4_csum;
1821                 uint64_t rx_bad_outer_l4_csum;
1822                 uint64_t rx_bad_outer_ip_csum;
1823         } ports_stats[RTE_MAX_ETHPORTS];
1824         uint64_t total_rx_dropped = 0;
1825         uint64_t total_tx_dropped = 0;
1826         uint64_t total_rx_nombuf = 0;
1827         struct rte_eth_stats stats;
1828         uint64_t fwd_cycles = 0;
1829         uint64_t total_recv = 0;
1830         uint64_t total_xmit = 0;
1831         struct rte_port *port;
1832         streamid_t sm_id;
1833         portid_t pt_id;
1834         int i;
1835
1836         memset(ports_stats, 0, sizeof(ports_stats));
1837
1838         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1839                 struct fwd_stream *fs = fwd_streams[sm_id];
1840
1841                 if (cur_fwd_config.nb_fwd_streams >
1842                     cur_fwd_config.nb_fwd_ports) {
1843                         fwd_stream_stats_display(sm_id);
1844                 } else {
1845                         ports_stats[fs->tx_port].tx_stream = fs;
1846                         ports_stats[fs->rx_port].rx_stream = fs;
1847                 }
1848
1849                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1850
1851                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1852                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1853                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1854                                 fs->rx_bad_outer_l4_csum;
1855                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
1856                                 fs->rx_bad_outer_ip_csum;
1857
1858                 if (record_core_cycles)
1859                         fwd_cycles += fs->core_cycles;
1860         }
1861         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1862                 pt_id = fwd_ports_ids[i];
1863                 port = &ports[pt_id];
1864
1865                 rte_eth_stats_get(pt_id, &stats);
1866                 stats.ipackets -= port->stats.ipackets;
1867                 stats.opackets -= port->stats.opackets;
1868                 stats.ibytes -= port->stats.ibytes;
1869                 stats.obytes -= port->stats.obytes;
1870                 stats.imissed -= port->stats.imissed;
1871                 stats.oerrors -= port->stats.oerrors;
1872                 stats.rx_nombuf -= port->stats.rx_nombuf;
1873
1874                 total_recv += stats.ipackets;
1875                 total_xmit += stats.opackets;
1876                 total_rx_dropped += stats.imissed;
1877                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1878                 total_tx_dropped += stats.oerrors;
1879                 total_rx_nombuf  += stats.rx_nombuf;
1880
1881                 printf("\n  %s Forward statistics for port %-2d %s\n",
1882                        fwd_stats_border, pt_id, fwd_stats_border);
1883
1884                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1885                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1886                        stats.ipackets + stats.imissed);
1887
1888                 if (cur_fwd_eng == &csum_fwd_engine) {
1889                         printf("  Bad-ipcsum: %-14"PRIu64
1890                                " Bad-l4csum: %-14"PRIu64
1891                                "Bad-outer-l4csum: %-14"PRIu64"\n",
1892                                ports_stats[pt_id].rx_bad_ip_csum,
1893                                ports_stats[pt_id].rx_bad_l4_csum,
1894                                ports_stats[pt_id].rx_bad_outer_l4_csum);
1895                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
1896                                ports_stats[pt_id].rx_bad_outer_ip_csum);
1897                 }
1898                 if (stats.ierrors + stats.rx_nombuf > 0) {
1899                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
1900                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1901                 }
1902
1903                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1904                        "TX-total: %-"PRIu64"\n",
1905                        stats.opackets, ports_stats[pt_id].tx_dropped,
1906                        stats.opackets + ports_stats[pt_id].tx_dropped);
1907
1908                 if (record_burst_stats) {
1909                         if (ports_stats[pt_id].rx_stream)
1910                                 pkt_burst_stats_display("RX",
1911                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1912                         if (ports_stats[pt_id].tx_stream)
1913                                 pkt_burst_stats_display("TX",
1914                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1915                 }
1916
1917                 printf("  %s--------------------------------%s\n",
1918                        fwd_stats_border, fwd_stats_border);
1919         }
1920
1921         printf("\n  %s Accumulated forward statistics for all ports"
1922                "%s\n",
1923                acc_stats_border, acc_stats_border);
1924         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1925                "%-"PRIu64"\n"
1926                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1927                "%-"PRIu64"\n",
1928                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1929                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1930         if (total_rx_nombuf > 0)
1931                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1932         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1933                "%s\n",
1934                acc_stats_border, acc_stats_border);
1935         if (record_core_cycles) {
1936 #define CYC_PER_MHZ 1E6
1937                 if (total_recv > 0 || total_xmit > 0) {
1938                         uint64_t total_pkts = 0;
1939                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1940                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1941                                 total_pkts = total_xmit;
1942                         else
1943                                 total_pkts = total_recv;
1944
1945                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1946                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1947                                " MHz Clock\n",
1948                                (double) fwd_cycles / total_pkts,
1949                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1950                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1951                 }
1952         }
1953 }
1954
1955 void
1956 fwd_stats_reset(void)
1957 {
1958         streamid_t sm_id;
1959         portid_t pt_id;
1960         int i;
1961
1962         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1963                 pt_id = fwd_ports_ids[i];
1964                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1965         }
1966         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1967                 struct fwd_stream *fs = fwd_streams[sm_id];
1968
1969                 fs->rx_packets = 0;
1970                 fs->tx_packets = 0;
1971                 fs->fwd_dropped = 0;
1972                 fs->rx_bad_ip_csum = 0;
1973                 fs->rx_bad_l4_csum = 0;
1974                 fs->rx_bad_outer_l4_csum = 0;
1975                 fs->rx_bad_outer_ip_csum = 0;
1976
1977                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1978                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1979                 fs->core_cycles = 0;
1980         }
1981 }
1982
1983 static void
1984 flush_fwd_rx_queues(void)
1985 {
1986         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1987         portid_t  rxp;
1988         portid_t port_id;
1989         queueid_t rxq;
1990         uint16_t  nb_rx;
1991         uint16_t  i;
1992         uint8_t   j;
1993         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1994         uint64_t timer_period;
1995
1996         /* convert to number of cycles */
1997         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1998
1999         for (j = 0; j < 2; j++) {
2000                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2001                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2002                                 port_id = fwd_ports_ids[rxp];
2003                                 /**
2004                                 * testpmd can stuck in the below do while loop
2005                                 * if rte_eth_rx_burst() always returns nonzero
2006                                 * packets. So timer is added to exit this loop
2007                                 * after 1sec timer expiry.
2008                                 */
2009                                 prev_tsc = rte_rdtsc();
2010                                 do {
2011                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2012                                                 pkts_burst, MAX_PKT_BURST);
2013                                         for (i = 0; i < nb_rx; i++)
2014                                                 rte_pktmbuf_free(pkts_burst[i]);
2015
2016                                         cur_tsc = rte_rdtsc();
2017                                         diff_tsc = cur_tsc - prev_tsc;
2018                                         timer_tsc += diff_tsc;
2019                                 } while ((nb_rx > 0) &&
2020                                         (timer_tsc < timer_period));
2021                                 timer_tsc = 0;
2022                         }
2023                 }
2024                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2025         }
2026 }
2027
2028 static void
2029 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2030 {
2031         struct fwd_stream **fsm;
2032         streamid_t nb_fs;
2033         streamid_t sm_id;
2034 #ifdef RTE_LIB_BITRATESTATS
2035         uint64_t tics_per_1sec;
2036         uint64_t tics_datum;
2037         uint64_t tics_current;
2038         uint16_t i, cnt_ports;
2039
2040         cnt_ports = nb_ports;
2041         tics_datum = rte_rdtsc();
2042         tics_per_1sec = rte_get_timer_hz();
2043 #endif
2044         fsm = &fwd_streams[fc->stream_idx];
2045         nb_fs = fc->stream_nb;
2046         do {
2047                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2048                         (*pkt_fwd)(fsm[sm_id]);
2049 #ifdef RTE_LIB_BITRATESTATS
2050                 if (bitrate_enabled != 0 &&
2051                                 bitrate_lcore_id == rte_lcore_id()) {
2052                         tics_current = rte_rdtsc();
2053                         if (tics_current - tics_datum >= tics_per_1sec) {
2054                                 /* Periodic bitrate calculation */
2055                                 for (i = 0; i < cnt_ports; i++)
2056                                         rte_stats_bitrate_calc(bitrate_data,
2057                                                 ports_ids[i]);
2058                                 tics_datum = tics_current;
2059                         }
2060                 }
2061 #endif
2062 #ifdef RTE_LIB_LATENCYSTATS
2063                 if (latencystats_enabled != 0 &&
2064                                 latencystats_lcore_id == rte_lcore_id())
2065                         rte_latencystats_update();
2066 #endif
2067
2068         } while (! fc->stopped);
2069 }
2070
2071 static int
2072 start_pkt_forward_on_core(void *fwd_arg)
2073 {
2074         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2075                              cur_fwd_config.fwd_eng->packet_fwd);
2076         return 0;
2077 }
2078
2079 /*
2080  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2081  * Used to start communication flows in network loopback test configurations.
2082  */
2083 static int
2084 run_one_txonly_burst_on_core(void *fwd_arg)
2085 {
2086         struct fwd_lcore *fwd_lc;
2087         struct fwd_lcore tmp_lcore;
2088
2089         fwd_lc = (struct fwd_lcore *) fwd_arg;
2090         tmp_lcore = *fwd_lc;
2091         tmp_lcore.stopped = 1;
2092         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2093         return 0;
2094 }
2095
2096 /*
2097  * Launch packet forwarding:
2098  *     - Setup per-port forwarding context.
2099  *     - launch logical cores with their forwarding configuration.
2100  */
2101 static void
2102 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2103 {
2104         port_fwd_begin_t port_fwd_begin;
2105         unsigned int i;
2106         unsigned int lc_id;
2107         int diag;
2108
2109         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2110         if (port_fwd_begin != NULL) {
2111                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2112                         (*port_fwd_begin)(fwd_ports_ids[i]);
2113         }
2114         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2115                 lc_id = fwd_lcores_cpuids[i];
2116                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2117                         fwd_lcores[i]->stopped = 0;
2118                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2119                                                      fwd_lcores[i], lc_id);
2120                         if (diag != 0)
2121                                 printf("launch lcore %u failed - diag=%d\n",
2122                                        lc_id, diag);
2123                 }
2124         }
2125 }
2126
2127 /*
2128  * Launch packet forwarding configuration.
2129  */
2130 void
2131 start_packet_forwarding(int with_tx_first)
2132 {
2133         port_fwd_begin_t port_fwd_begin;
2134         port_fwd_end_t  port_fwd_end;
2135         unsigned int i;
2136
2137         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2138                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2139
2140         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2141                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2142
2143         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2144                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2145                 (!nb_rxq || !nb_txq))
2146                 rte_exit(EXIT_FAILURE,
2147                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2148                         cur_fwd_eng->fwd_mode_name);
2149
2150         if (all_ports_started() == 0) {
2151                 printf("Not all ports were started\n");
2152                 return;
2153         }
2154         if (test_done == 0) {
2155                 printf("Packet forwarding already started\n");
2156                 return;
2157         }
2158         test_done = 0;
2159
2160         fwd_config_setup();
2161
2162         if(!no_flush_rx)
2163                 flush_fwd_rx_queues();
2164
2165         pkt_fwd_config_display(&cur_fwd_config);
2166         rxtx_config_display();
2167
2168         fwd_stats_reset();
2169         if (with_tx_first) {
2170                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2171                 if (port_fwd_begin != NULL) {
2172                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2173                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2174                 }
2175                 while (with_tx_first--) {
2176                         launch_packet_forwarding(
2177                                         run_one_txonly_burst_on_core);
2178                         rte_eal_mp_wait_lcore();
2179                 }
2180                 port_fwd_end = tx_only_engine.port_fwd_end;
2181                 if (port_fwd_end != NULL) {
2182                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2183                                 (*port_fwd_end)(fwd_ports_ids[i]);
2184                 }
2185         }
2186         launch_packet_forwarding(start_pkt_forward_on_core);
2187 }
2188
2189 void
2190 stop_packet_forwarding(void)
2191 {
2192         port_fwd_end_t port_fwd_end;
2193         lcoreid_t lc_id;
2194         portid_t pt_id;
2195         int i;
2196
2197         if (test_done) {
2198                 printf("Packet forwarding not started\n");
2199                 return;
2200         }
2201         printf("Telling cores to stop...");
2202         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2203                 fwd_lcores[lc_id]->stopped = 1;
2204         printf("\nWaiting for lcores to finish...\n");
2205         rte_eal_mp_wait_lcore();
2206         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2207         if (port_fwd_end != NULL) {
2208                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2209                         pt_id = fwd_ports_ids[i];
2210                         (*port_fwd_end)(pt_id);
2211                 }
2212         }
2213
2214         fwd_stats_display();
2215
2216         printf("\nDone.\n");
2217         test_done = 1;
2218 }
2219
2220 void
2221 dev_set_link_up(portid_t pid)
2222 {
2223         if (rte_eth_dev_set_link_up(pid) < 0)
2224                 printf("\nSet link up fail.\n");
2225 }
2226
2227 void
2228 dev_set_link_down(portid_t pid)
2229 {
2230         if (rte_eth_dev_set_link_down(pid) < 0)
2231                 printf("\nSet link down fail.\n");
2232 }
2233
2234 static int
2235 all_ports_started(void)
2236 {
2237         portid_t pi;
2238         struct rte_port *port;
2239
2240         RTE_ETH_FOREACH_DEV(pi) {
2241                 port = &ports[pi];
2242                 /* Check if there is a port which is not started */
2243                 if ((port->port_status != RTE_PORT_STARTED) &&
2244                         (port->slave_flag == 0))
2245                         return 0;
2246         }
2247
2248         /* No port is not started */
2249         return 1;
2250 }
2251
2252 int
2253 port_is_stopped(portid_t port_id)
2254 {
2255         struct rte_port *port = &ports[port_id];
2256
2257         if ((port->port_status != RTE_PORT_STOPPED) &&
2258             (port->slave_flag == 0))
2259                 return 0;
2260         return 1;
2261 }
2262
2263 int
2264 all_ports_stopped(void)
2265 {
2266         portid_t pi;
2267
2268         RTE_ETH_FOREACH_DEV(pi) {
2269                 if (!port_is_stopped(pi))
2270                         return 0;
2271         }
2272
2273         return 1;
2274 }
2275
2276 int
2277 port_is_started(portid_t port_id)
2278 {
2279         if (port_id_is_invalid(port_id, ENABLED_WARN))
2280                 return 0;
2281
2282         if (ports[port_id].port_status != RTE_PORT_STARTED)
2283                 return 0;
2284
2285         return 1;
2286 }
2287
2288 /* Configure the Rx and Tx hairpin queues for the selected port. */
2289 static int
2290 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2291 {
2292         queueid_t qi;
2293         struct rte_eth_hairpin_conf hairpin_conf = {
2294                 .peer_count = 1,
2295         };
2296         int i;
2297         int diag;
2298         struct rte_port *port = &ports[pi];
2299         uint16_t peer_rx_port = pi;
2300         uint16_t peer_tx_port = pi;
2301         uint32_t manual = 1;
2302         uint32_t tx_exp = hairpin_mode & 0x10;
2303
2304         if (!(hairpin_mode & 0xf)) {
2305                 peer_rx_port = pi;
2306                 peer_tx_port = pi;
2307                 manual = 0;
2308         } else if (hairpin_mode & 0x1) {
2309                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2310                                                        RTE_ETH_DEV_NO_OWNER);
2311                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2312                         peer_tx_port = rte_eth_find_next_owned_by(0,
2313                                                 RTE_ETH_DEV_NO_OWNER);
2314                 if (p_pi != RTE_MAX_ETHPORTS) {
2315                         peer_rx_port = p_pi;
2316                 } else {
2317                         uint16_t next_pi;
2318
2319                         /* Last port will be the peer RX port of the first. */
2320                         RTE_ETH_FOREACH_DEV(next_pi)
2321                                 peer_rx_port = next_pi;
2322                 }
2323                 manual = 1;
2324         } else if (hairpin_mode & 0x2) {
2325                 if (cnt_pi & 0x1) {
2326                         peer_rx_port = p_pi;
2327                 } else {
2328                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2329                                                 RTE_ETH_DEV_NO_OWNER);
2330                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2331                                 peer_rx_port = pi;
2332                 }
2333                 peer_tx_port = peer_rx_port;
2334                 manual = 1;
2335         }
2336
2337         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2338                 hairpin_conf.peers[0].port = peer_rx_port;
2339                 hairpin_conf.peers[0].queue = i + nb_rxq;
2340                 hairpin_conf.manual_bind = !!manual;
2341                 hairpin_conf.tx_explicit = !!tx_exp;
2342                 diag = rte_eth_tx_hairpin_queue_setup
2343                         (pi, qi, nb_txd, &hairpin_conf);
2344                 i++;
2345                 if (diag == 0)
2346                         continue;
2347
2348                 /* Fail to setup rx queue, return */
2349                 if (rte_atomic16_cmpset(&(port->port_status),
2350                                         RTE_PORT_HANDLING,
2351                                         RTE_PORT_STOPPED) == 0)
2352                         printf("Port %d can not be set back "
2353                                         "to stopped\n", pi);
2354                 printf("Fail to configure port %d hairpin "
2355                                 "queues\n", pi);
2356                 /* try to reconfigure queues next time */
2357                 port->need_reconfig_queues = 1;
2358                 return -1;
2359         }
2360         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2361                 hairpin_conf.peers[0].port = peer_tx_port;
2362                 hairpin_conf.peers[0].queue = i + nb_txq;
2363                 hairpin_conf.manual_bind = !!manual;
2364                 hairpin_conf.tx_explicit = !!tx_exp;
2365                 diag = rte_eth_rx_hairpin_queue_setup
2366                         (pi, qi, nb_rxd, &hairpin_conf);
2367                 i++;
2368                 if (diag == 0)
2369                         continue;
2370
2371                 /* Fail to setup rx queue, return */
2372                 if (rte_atomic16_cmpset(&(port->port_status),
2373                                         RTE_PORT_HANDLING,
2374                                         RTE_PORT_STOPPED) == 0)
2375                         printf("Port %d can not be set back "
2376                                         "to stopped\n", pi);
2377                 printf("Fail to configure port %d hairpin "
2378                                 "queues\n", pi);
2379                 /* try to reconfigure queues next time */
2380                 port->need_reconfig_queues = 1;
2381                 return -1;
2382         }
2383         return 0;
2384 }
2385
2386 /* Configure the Rx with optional split. */
2387 int
2388 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2389                uint16_t nb_rx_desc, unsigned int socket_id,
2390                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2391 {
2392         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2393         unsigned int i, mp_n;
2394         int ret;
2395
2396         if (rx_pkt_nb_segs <= 1 ||
2397             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2398                 rx_conf->rx_seg = NULL;
2399                 rx_conf->rx_nseg = 0;
2400                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2401                                              nb_rx_desc, socket_id,
2402                                              rx_conf, mp);
2403                 return ret;
2404         }
2405         for (i = 0; i < rx_pkt_nb_segs; i++) {
2406                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2407                 struct rte_mempool *mpx;
2408                 /*
2409                  * Use last valid pool for the segments with number
2410                  * exceeding the pool index.
2411                  */
2412                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2413                 mpx = mbuf_pool_find(socket_id, mp_n);
2414                 /* Handle zero as mbuf data buffer size. */
2415                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2416                                    rx_pkt_seg_lengths[i] :
2417                                    mbuf_data_size[mp_n];
2418                 rx_seg->offset = i < rx_pkt_nb_offs ?
2419                                    rx_pkt_seg_offsets[i] : 0;
2420                 rx_seg->mp = mpx ? mpx : mp;
2421         }
2422         rx_conf->rx_nseg = rx_pkt_nb_segs;
2423         rx_conf->rx_seg = rx_useg;
2424         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2425                                     socket_id, rx_conf, NULL);
2426         rx_conf->rx_seg = NULL;
2427         rx_conf->rx_nseg = 0;
2428         return ret;
2429 }
2430
2431 int
2432 start_port(portid_t pid)
2433 {
2434         int diag, need_check_link_status = -1;
2435         portid_t pi;
2436         portid_t p_pi = RTE_MAX_ETHPORTS;
2437         portid_t pl[RTE_MAX_ETHPORTS];
2438         portid_t peer_pl[RTE_MAX_ETHPORTS];
2439         uint16_t cnt_pi = 0;
2440         uint16_t cfg_pi = 0;
2441         int peer_pi;
2442         queueid_t qi;
2443         struct rte_port *port;
2444         struct rte_ether_addr mac_addr;
2445         struct rte_eth_hairpin_cap cap;
2446
2447         if (port_id_is_invalid(pid, ENABLED_WARN))
2448                 return 0;
2449
2450         RTE_ETH_FOREACH_DEV(pi) {
2451                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2452                         continue;
2453
2454                 need_check_link_status = 0;
2455                 port = &ports[pi];
2456                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2457                                                  RTE_PORT_HANDLING) == 0) {
2458                         printf("Port %d is now not stopped\n", pi);
2459                         continue;
2460                 }
2461
2462                 if (port->need_reconfig > 0) {
2463                         port->need_reconfig = 0;
2464
2465                         if (flow_isolate_all) {
2466                                 int ret = port_flow_isolate(pi, 1);
2467                                 if (ret) {
2468                                         printf("Failed to apply isolated"
2469                                                " mode on port %d\n", pi);
2470                                         return -1;
2471                                 }
2472                         }
2473                         configure_rxtx_dump_callbacks(0);
2474                         printf("Configuring Port %d (socket %u)\n", pi,
2475                                         port->socket_id);
2476                         if (nb_hairpinq > 0 &&
2477                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2478                                 printf("Port %d doesn't support hairpin "
2479                                        "queues\n", pi);
2480                                 return -1;
2481                         }
2482                         /* configure port */
2483                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2484                                                      nb_txq + nb_hairpinq,
2485                                                      &(port->dev_conf));
2486                         if (diag != 0) {
2487                                 if (rte_atomic16_cmpset(&(port->port_status),
2488                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2489                                         printf("Port %d can not be set back "
2490                                                         "to stopped\n", pi);
2491                                 printf("Fail to configure port %d\n", pi);
2492                                 /* try to reconfigure port next time */
2493                                 port->need_reconfig = 1;
2494                                 return -1;
2495                         }
2496                 }
2497                 if (port->need_reconfig_queues > 0) {
2498                         port->need_reconfig_queues = 0;
2499                         /* setup tx queues */
2500                         for (qi = 0; qi < nb_txq; qi++) {
2501                                 if ((numa_support) &&
2502                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2503                                         diag = rte_eth_tx_queue_setup(pi, qi,
2504                                                 port->nb_tx_desc[qi],
2505                                                 txring_numa[pi],
2506                                                 &(port->tx_conf[qi]));
2507                                 else
2508                                         diag = rte_eth_tx_queue_setup(pi, qi,
2509                                                 port->nb_tx_desc[qi],
2510                                                 port->socket_id,
2511                                                 &(port->tx_conf[qi]));
2512
2513                                 if (diag == 0)
2514                                         continue;
2515
2516                                 /* Fail to setup tx queue, return */
2517                                 if (rte_atomic16_cmpset(&(port->port_status),
2518                                                         RTE_PORT_HANDLING,
2519                                                         RTE_PORT_STOPPED) == 0)
2520                                         printf("Port %d can not be set back "
2521                                                         "to stopped\n", pi);
2522                                 printf("Fail to configure port %d tx queues\n",
2523                                        pi);
2524                                 /* try to reconfigure queues next time */
2525                                 port->need_reconfig_queues = 1;
2526                                 return -1;
2527                         }
2528                         for (qi = 0; qi < nb_rxq; qi++) {
2529                                 /* setup rx queues */
2530                                 if ((numa_support) &&
2531                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2532                                         struct rte_mempool * mp =
2533                                                 mbuf_pool_find
2534                                                         (rxring_numa[pi], 0);
2535                                         if (mp == NULL) {
2536                                                 printf("Failed to setup RX queue:"
2537                                                         "No mempool allocation"
2538                                                         " on the socket %d\n",
2539                                                         rxring_numa[pi]);
2540                                                 return -1;
2541                                         }
2542
2543                                         diag = rx_queue_setup(pi, qi,
2544                                              port->nb_rx_desc[qi],
2545                                              rxring_numa[pi],
2546                                              &(port->rx_conf[qi]),
2547                                              mp);
2548                                 } else {
2549                                         struct rte_mempool *mp =
2550                                                 mbuf_pool_find
2551                                                         (port->socket_id, 0);
2552                                         if (mp == NULL) {
2553                                                 printf("Failed to setup RX queue:"
2554                                                         "No mempool allocation"
2555                                                         " on the socket %d\n",
2556                                                         port->socket_id);
2557                                                 return -1;
2558                                         }
2559                                         diag = rx_queue_setup(pi, qi,
2560                                              port->nb_rx_desc[qi],
2561                                              port->socket_id,
2562                                              &(port->rx_conf[qi]),
2563                                              mp);
2564                                 }
2565                                 if (diag == 0)
2566                                         continue;
2567
2568                                 /* Fail to setup rx queue, return */
2569                                 if (rte_atomic16_cmpset(&(port->port_status),
2570                                                         RTE_PORT_HANDLING,
2571                                                         RTE_PORT_STOPPED) == 0)
2572                                         printf("Port %d can not be set back "
2573                                                         "to stopped\n", pi);
2574                                 printf("Fail to configure port %d rx queues\n",
2575                                        pi);
2576                                 /* try to reconfigure queues next time */
2577                                 port->need_reconfig_queues = 1;
2578                                 return -1;
2579                         }
2580                         /* setup hairpin queues */
2581                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2582                                 return -1;
2583                 }
2584                 configure_rxtx_dump_callbacks(verbose_level);
2585                 if (clear_ptypes) {
2586                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2587                                         NULL, 0);
2588                         if (diag < 0)
2589                                 printf(
2590                                 "Port %d: Failed to disable Ptype parsing\n",
2591                                 pi);
2592                 }
2593
2594                 p_pi = pi;
2595                 cnt_pi++;
2596
2597                 /* start port */
2598                 diag = rte_eth_dev_start(pi);
2599                 if (diag < 0) {
2600                         printf("Fail to start port %d: %s\n", pi,
2601                                rte_strerror(-diag));
2602
2603                         /* Fail to setup rx queue, return */
2604                         if (rte_atomic16_cmpset(&(port->port_status),
2605                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2606                                 printf("Port %d can not be set back to "
2607                                                         "stopped\n", pi);
2608                         continue;
2609                 }
2610
2611                 if (rte_atomic16_cmpset(&(port->port_status),
2612                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2613                         printf("Port %d can not be set into started\n", pi);
2614
2615                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2616                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2617                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2618                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2619                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2620
2621                 /* at least one port started, need checking link status */
2622                 need_check_link_status = 1;
2623
2624                 pl[cfg_pi++] = pi;
2625         }
2626
2627         if (need_check_link_status == 1 && !no_link_check)
2628                 check_all_ports_link_status(RTE_PORT_ALL);
2629         else if (need_check_link_status == 0)
2630                 printf("Please stop the ports first\n");
2631
2632         if (hairpin_mode & 0xf) {
2633                 uint16_t i;
2634                 int j;
2635
2636                 /* bind all started hairpin ports */
2637                 for (i = 0; i < cfg_pi; i++) {
2638                         pi = pl[i];
2639                         /* bind current Tx to all peer Rx */
2640                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2641                                                         RTE_MAX_ETHPORTS, 1);
2642                         if (peer_pi < 0)
2643                                 return peer_pi;
2644                         for (j = 0; j < peer_pi; j++) {
2645                                 if (!port_is_started(peer_pl[j]))
2646                                         continue;
2647                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2648                                 if (diag < 0) {
2649                                         printf("Error during binding hairpin"
2650                                                " Tx port %u to %u: %s\n",
2651                                                pi, peer_pl[j],
2652                                                rte_strerror(-diag));
2653                                         return -1;
2654                                 }
2655                         }
2656                         /* bind all peer Tx to current Rx */
2657                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2658                                                         RTE_MAX_ETHPORTS, 0);
2659                         if (peer_pi < 0)
2660                                 return peer_pi;
2661                         for (j = 0; j < peer_pi; j++) {
2662                                 if (!port_is_started(peer_pl[j]))
2663                                         continue;
2664                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2665                                 if (diag < 0) {
2666                                         printf("Error during binding hairpin"
2667                                                " Tx port %u to %u: %s\n",
2668                                                peer_pl[j], pi,
2669                                                rte_strerror(-diag));
2670                                         return -1;
2671                                 }
2672                         }
2673                 }
2674         }
2675
2676         printf("Done\n");
2677         return 0;
2678 }
2679
2680 void
2681 stop_port(portid_t pid)
2682 {
2683         portid_t pi;
2684         struct rte_port *port;
2685         int need_check_link_status = 0;
2686         portid_t peer_pl[RTE_MAX_ETHPORTS];
2687         int peer_pi;
2688
2689         if (port_id_is_invalid(pid, ENABLED_WARN))
2690                 return;
2691
2692         printf("Stopping ports...\n");
2693
2694         RTE_ETH_FOREACH_DEV(pi) {
2695                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2696                         continue;
2697
2698                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2699                         printf("Please remove port %d from forwarding configuration.\n", pi);
2700                         continue;
2701                 }
2702
2703                 if (port_is_bonding_slave(pi)) {
2704                         printf("Please remove port %d from bonded device.\n", pi);
2705                         continue;
2706                 }
2707
2708                 port = &ports[pi];
2709                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2710                                                 RTE_PORT_HANDLING) == 0)
2711                         continue;
2712
2713                 if (hairpin_mode & 0xf) {
2714                         int j;
2715
2716                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2717                         /* unbind all peer Tx from current Rx */
2718                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2719                                                         RTE_MAX_ETHPORTS, 0);
2720                         if (peer_pi < 0)
2721                                 continue;
2722                         for (j = 0; j < peer_pi; j++) {
2723                                 if (!port_is_started(peer_pl[j]))
2724                                         continue;
2725                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2726                         }
2727                 }
2728
2729                 if (port->flow_list)
2730                         port_flow_flush(pi);
2731
2732                 if (rte_eth_dev_stop(pi) != 0)
2733                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2734                                 pi);
2735
2736                 if (rte_atomic16_cmpset(&(port->port_status),
2737                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2738                         printf("Port %d can not be set into stopped\n", pi);
2739                 need_check_link_status = 1;
2740         }
2741         if (need_check_link_status && !no_link_check)
2742                 check_all_ports_link_status(RTE_PORT_ALL);
2743
2744         printf("Done\n");
2745 }
2746
2747 static void
2748 remove_invalid_ports_in(portid_t *array, portid_t *total)
2749 {
2750         portid_t i;
2751         portid_t new_total = 0;
2752
2753         for (i = 0; i < *total; i++)
2754                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2755                         array[new_total] = array[i];
2756                         new_total++;
2757                 }
2758         *total = new_total;
2759 }
2760
2761 static void
2762 remove_invalid_ports(void)
2763 {
2764         remove_invalid_ports_in(ports_ids, &nb_ports);
2765         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2766         nb_cfg_ports = nb_fwd_ports;
2767 }
2768
2769 void
2770 close_port(portid_t pid)
2771 {
2772         portid_t pi;
2773         struct rte_port *port;
2774
2775         if (port_id_is_invalid(pid, ENABLED_WARN))
2776                 return;
2777
2778         printf("Closing ports...\n");
2779
2780         RTE_ETH_FOREACH_DEV(pi) {
2781                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2782                         continue;
2783
2784                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2785                         printf("Please remove port %d from forwarding configuration.\n", pi);
2786                         continue;
2787                 }
2788
2789                 if (port_is_bonding_slave(pi)) {
2790                         printf("Please remove port %d from bonded device.\n", pi);
2791                         continue;
2792                 }
2793
2794                 port = &ports[pi];
2795                 if (rte_atomic16_cmpset(&(port->port_status),
2796                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2797                         printf("Port %d is already closed\n", pi);
2798                         continue;
2799                 }
2800
2801                 port_flow_flush(pi);
2802                 rte_eth_dev_close(pi);
2803         }
2804
2805         remove_invalid_ports();
2806         printf("Done\n");
2807 }
2808
2809 void
2810 reset_port(portid_t pid)
2811 {
2812         int diag;
2813         portid_t pi;
2814         struct rte_port *port;
2815
2816         if (port_id_is_invalid(pid, ENABLED_WARN))
2817                 return;
2818
2819         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2820                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2821                 printf("Can not reset port(s), please stop port(s) first.\n");
2822                 return;
2823         }
2824
2825         printf("Resetting ports...\n");
2826
2827         RTE_ETH_FOREACH_DEV(pi) {
2828                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2829                         continue;
2830
2831                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2832                         printf("Please remove port %d from forwarding "
2833                                "configuration.\n", pi);
2834                         continue;
2835                 }
2836
2837                 if (port_is_bonding_slave(pi)) {
2838                         printf("Please remove port %d from bonded device.\n",
2839                                pi);
2840                         continue;
2841                 }
2842
2843                 diag = rte_eth_dev_reset(pi);
2844                 if (diag == 0) {
2845                         port = &ports[pi];
2846                         port->need_reconfig = 1;
2847                         port->need_reconfig_queues = 1;
2848                 } else {
2849                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2850                 }
2851         }
2852
2853         printf("Done\n");
2854 }
2855
2856 void
2857 attach_port(char *identifier)
2858 {
2859         portid_t pi;
2860         struct rte_dev_iterator iterator;
2861
2862         printf("Attaching a new port...\n");
2863
2864         if (identifier == NULL) {
2865                 printf("Invalid parameters are specified\n");
2866                 return;
2867         }
2868
2869         if (rte_dev_probe(identifier) < 0) {
2870                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2871                 return;
2872         }
2873
2874         /* first attach mode: event */
2875         if (setup_on_probe_event) {
2876                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2877                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2878                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2879                                         ports[pi].need_setup != 0)
2880                                 setup_attached_port(pi);
2881                 return;
2882         }
2883
2884         /* second attach mode: iterator */
2885         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2886                 /* setup ports matching the devargs used for probing */
2887                 if (port_is_forwarding(pi))
2888                         continue; /* port was already attached before */
2889                 setup_attached_port(pi);
2890         }
2891 }
2892
2893 static void
2894 setup_attached_port(portid_t pi)
2895 {
2896         unsigned int socket_id;
2897         int ret;
2898
2899         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2900         /* if socket_id is invalid, set to the first available socket. */
2901         if (check_socket_id(socket_id) < 0)
2902                 socket_id = socket_ids[0];
2903         reconfig(pi, socket_id);
2904         ret = rte_eth_promiscuous_enable(pi);
2905         if (ret != 0)
2906                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2907                         pi, rte_strerror(-ret));
2908
2909         ports_ids[nb_ports++] = pi;
2910         fwd_ports_ids[nb_fwd_ports++] = pi;
2911         nb_cfg_ports = nb_fwd_ports;
2912         ports[pi].need_setup = 0;
2913         ports[pi].port_status = RTE_PORT_STOPPED;
2914
2915         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2916         printf("Done\n");
2917 }
2918
2919 static void
2920 detach_device(struct rte_device *dev)
2921 {
2922         portid_t sibling;
2923
2924         if (dev == NULL) {
2925                 printf("Device already removed\n");
2926                 return;
2927         }
2928
2929         printf("Removing a device...\n");
2930
2931         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2932                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2933                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2934                                 printf("Port %u not stopped\n", sibling);
2935                                 return;
2936                         }
2937                         port_flow_flush(sibling);
2938                 }
2939         }
2940
2941         if (rte_dev_remove(dev) < 0) {
2942                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2943                 return;
2944         }
2945         remove_invalid_ports();
2946
2947         printf("Device is detached\n");
2948         printf("Now total ports is %d\n", nb_ports);
2949         printf("Done\n");
2950         return;
2951 }
2952
2953 void
2954 detach_port_device(portid_t port_id)
2955 {
2956         if (port_id_is_invalid(port_id, ENABLED_WARN))
2957                 return;
2958
2959         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2960                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2961                         printf("Port not stopped\n");
2962                         return;
2963                 }
2964                 printf("Port was not closed\n");
2965         }
2966
2967         detach_device(rte_eth_devices[port_id].device);
2968 }
2969
2970 void
2971 detach_devargs(char *identifier)
2972 {
2973         struct rte_dev_iterator iterator;
2974         struct rte_devargs da;
2975         portid_t port_id;
2976
2977         printf("Removing a device...\n");
2978
2979         memset(&da, 0, sizeof(da));
2980         if (rte_devargs_parsef(&da, "%s", identifier)) {
2981                 printf("cannot parse identifier\n");
2982                 return;
2983         }
2984
2985         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2986                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2987                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2988                                 printf("Port %u not stopped\n", port_id);
2989                                 rte_eth_iterator_cleanup(&iterator);
2990                                 rte_devargs_reset(&da);
2991                                 return;
2992                         }
2993                         port_flow_flush(port_id);
2994                 }
2995         }
2996
2997         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2998                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2999                             da.name, da.bus->name);
3000                 rte_devargs_reset(&da);
3001                 return;
3002         }
3003
3004         remove_invalid_ports();
3005
3006         printf("Device %s is detached\n", identifier);
3007         printf("Now total ports is %d\n", nb_ports);
3008         printf("Done\n");
3009         rte_devargs_reset(&da);
3010 }
3011
3012 void
3013 pmd_test_exit(void)
3014 {
3015         portid_t pt_id;
3016         unsigned int i;
3017         int ret;
3018
3019         if (test_done == 0)
3020                 stop_packet_forwarding();
3021
3022 #ifndef RTE_EXEC_ENV_WINDOWS
3023         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3024                 if (mempools[i]) {
3025                         if (mp_alloc_type == MP_ALLOC_ANON)
3026                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3027                                                      NULL);
3028                 }
3029         }
3030 #endif
3031         if (ports != NULL) {
3032                 no_link_check = 1;
3033                 RTE_ETH_FOREACH_DEV(pt_id) {
3034                         printf("\nStopping port %d...\n", pt_id);
3035                         fflush(stdout);
3036                         stop_port(pt_id);
3037                 }
3038                 RTE_ETH_FOREACH_DEV(pt_id) {
3039                         printf("\nShutting down port %d...\n", pt_id);
3040                         fflush(stdout);
3041                         close_port(pt_id);
3042                 }
3043         }
3044
3045         if (hot_plug) {
3046                 ret = rte_dev_event_monitor_stop();
3047                 if (ret) {
3048                         RTE_LOG(ERR, EAL,
3049                                 "fail to stop device event monitor.");
3050                         return;
3051                 }
3052
3053                 ret = rte_dev_event_callback_unregister(NULL,
3054                         dev_event_callback, NULL);
3055                 if (ret < 0) {
3056                         RTE_LOG(ERR, EAL,
3057                                 "fail to unregister device event callback.\n");
3058                         return;
3059                 }
3060
3061                 ret = rte_dev_hotplug_handle_disable();
3062                 if (ret) {
3063                         RTE_LOG(ERR, EAL,
3064                                 "fail to disable hotplug handling.\n");
3065                         return;
3066                 }
3067         }
3068         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3069                 if (mempools[i])
3070                         rte_mempool_free(mempools[i]);
3071         }
3072
3073         printf("\nBye...\n");
3074 }
3075
3076 typedef void (*cmd_func_t)(void);
3077 struct pmd_test_command {
3078         const char *cmd_name;
3079         cmd_func_t cmd_func;
3080 };
3081
3082 /* Check the link status of all ports in up to 9s, and print them finally */
3083 static void
3084 check_all_ports_link_status(uint32_t port_mask)
3085 {
3086 #define CHECK_INTERVAL 100 /* 100ms */
3087 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3088         portid_t portid;
3089         uint8_t count, all_ports_up, print_flag = 0;
3090         struct rte_eth_link link;
3091         int ret;
3092         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3093
3094         printf("Checking link statuses...\n");
3095         fflush(stdout);
3096         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3097                 all_ports_up = 1;
3098                 RTE_ETH_FOREACH_DEV(portid) {
3099                         if ((port_mask & (1 << portid)) == 0)
3100                                 continue;
3101                         memset(&link, 0, sizeof(link));
3102                         ret = rte_eth_link_get_nowait(portid, &link);
3103                         if (ret < 0) {
3104                                 all_ports_up = 0;
3105                                 if (print_flag == 1)
3106                                         printf("Port %u link get failed: %s\n",
3107                                                 portid, rte_strerror(-ret));
3108                                 continue;
3109                         }
3110                         /* print link status if flag set */
3111                         if (print_flag == 1) {
3112                                 rte_eth_link_to_str(link_status,
3113                                         sizeof(link_status), &link);
3114                                 printf("Port %d %s\n", portid, link_status);
3115                                 continue;
3116                         }
3117                         /* clear all_ports_up flag if any link down */
3118                         if (link.link_status == ETH_LINK_DOWN) {
3119                                 all_ports_up = 0;
3120                                 break;
3121                         }
3122                 }
3123                 /* after finally printing all link status, get out */
3124                 if (print_flag == 1)
3125                         break;
3126
3127                 if (all_ports_up == 0) {
3128                         fflush(stdout);
3129                         rte_delay_ms(CHECK_INTERVAL);
3130                 }
3131
3132                 /* set the print_flag if all ports up or timeout */
3133                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3134                         print_flag = 1;
3135                 }
3136
3137                 if (lsc_interrupt)
3138                         break;
3139         }
3140 }
3141
3142 static void
3143 rmv_port_callback(void *arg)
3144 {
3145         int need_to_start = 0;
3146         int org_no_link_check = no_link_check;
3147         portid_t port_id = (intptr_t)arg;
3148         struct rte_device *dev;
3149
3150         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3151
3152         if (!test_done && port_is_forwarding(port_id)) {
3153                 need_to_start = 1;
3154                 stop_packet_forwarding();
3155         }
3156         no_link_check = 1;
3157         stop_port(port_id);
3158         no_link_check = org_no_link_check;
3159
3160         /* Save rte_device pointer before closing ethdev port */
3161         dev = rte_eth_devices[port_id].device;
3162         close_port(port_id);
3163         detach_device(dev); /* might be already removed or have more ports */
3164
3165         if (need_to_start)
3166                 start_packet_forwarding(0);
3167 }
3168
3169 /* This function is used by the interrupt thread */
3170 static int
3171 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3172                   void *ret_param)
3173 {
3174         RTE_SET_USED(param);
3175         RTE_SET_USED(ret_param);
3176
3177         if (type >= RTE_ETH_EVENT_MAX) {
3178                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3179                         port_id, __func__, type);
3180                 fflush(stderr);
3181         } else if (event_print_mask & (UINT32_C(1) << type)) {
3182                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3183                         eth_event_desc[type]);
3184                 fflush(stdout);
3185         }
3186
3187         switch (type) {
3188         case RTE_ETH_EVENT_NEW:
3189                 ports[port_id].need_setup = 1;
3190                 ports[port_id].port_status = RTE_PORT_HANDLING;
3191                 break;
3192         case RTE_ETH_EVENT_INTR_RMV:
3193                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3194                         break;
3195                 if (rte_eal_alarm_set(100000,
3196                                 rmv_port_callback, (void *)(intptr_t)port_id))
3197                         fprintf(stderr, "Could not set up deferred device removal\n");
3198                 break;
3199         case RTE_ETH_EVENT_DESTROY:
3200                 ports[port_id].port_status = RTE_PORT_CLOSED;
3201                 printf("Port %u is closed\n", port_id);
3202                 break;
3203         default:
3204                 break;
3205         }
3206         return 0;
3207 }
3208
3209 static int
3210 register_eth_event_callback(void)
3211 {
3212         int ret;
3213         enum rte_eth_event_type event;
3214
3215         for (event = RTE_ETH_EVENT_UNKNOWN;
3216                         event < RTE_ETH_EVENT_MAX; event++) {
3217                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3218                                 event,
3219                                 eth_event_callback,
3220                                 NULL);
3221                 if (ret != 0) {
3222                         TESTPMD_LOG(ERR, "Failed to register callback for "
3223                                         "%s event\n", eth_event_desc[event]);
3224                         return -1;
3225                 }
3226         }
3227
3228         return 0;
3229 }
3230
3231 /* This function is used by the interrupt thread */
3232 static void
3233 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3234                              __rte_unused void *arg)
3235 {
3236         uint16_t port_id;
3237         int ret;
3238
3239         if (type >= RTE_DEV_EVENT_MAX) {
3240                 fprintf(stderr, "%s called upon invalid event %d\n",
3241                         __func__, type);
3242                 fflush(stderr);
3243         }
3244
3245         switch (type) {
3246         case RTE_DEV_EVENT_REMOVE:
3247                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3248                         device_name);
3249                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3250                 if (ret) {
3251                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3252                                 device_name);
3253                         return;
3254                 }
3255                 /*
3256                  * Because the user's callback is invoked in eal interrupt
3257                  * callback, the interrupt callback need to be finished before
3258                  * it can be unregistered when detaching device. So finish
3259                  * callback soon and use a deferred removal to detach device
3260                  * is need. It is a workaround, once the device detaching be
3261                  * moved into the eal in the future, the deferred removal could
3262                  * be deleted.
3263                  */
3264                 if (rte_eal_alarm_set(100000,
3265                                 rmv_port_callback, (void *)(intptr_t)port_id))
3266                         RTE_LOG(ERR, EAL,
3267                                 "Could not set up deferred device removal\n");
3268                 break;
3269         case RTE_DEV_EVENT_ADD:
3270                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3271                         device_name);
3272                 /* TODO: After finish kernel driver binding,
3273                  * begin to attach port.
3274                  */
3275                 break;
3276         default:
3277                 break;
3278         }
3279 }
3280
3281 static void
3282 rxtx_port_config(struct rte_port *port)
3283 {
3284         uint16_t qid;
3285         uint64_t offloads;
3286
3287         for (qid = 0; qid < nb_rxq; qid++) {
3288                 offloads = port->rx_conf[qid].offloads;
3289                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3290                 if (offloads != 0)
3291                         port->rx_conf[qid].offloads = offloads;
3292
3293                 /* Check if any Rx parameters have been passed */
3294                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3295                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3296
3297                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3298                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3299
3300                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3301                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3302
3303                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3304                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3305
3306                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3307                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3308
3309                 port->nb_rx_desc[qid] = nb_rxd;
3310         }
3311
3312         for (qid = 0; qid < nb_txq; qid++) {
3313                 offloads = port->tx_conf[qid].offloads;
3314                 port->tx_conf[qid] = port->dev_info.default_txconf;
3315                 if (offloads != 0)
3316                         port->tx_conf[qid].offloads = offloads;
3317
3318                 /* Check if any Tx parameters have been passed */
3319                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3320                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3321
3322                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3323                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3324
3325                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3326                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3327
3328                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3329                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3330
3331                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3332                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3333
3334                 port->nb_tx_desc[qid] = nb_txd;
3335         }
3336 }
3337
3338 /*
3339  * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3340  * MTU is also aligned if JUMBO_FRAME offload is not set.
3341  *
3342  * port->dev_info should be set before calling this function.
3343  *
3344  * return 0 on success, negative on error
3345  */
3346 int
3347 update_jumbo_frame_offload(portid_t portid)
3348 {
3349         struct rte_port *port = &ports[portid];
3350         uint32_t eth_overhead;
3351         uint64_t rx_offloads;
3352         int ret;
3353         bool on;
3354
3355         /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3356         if (port->dev_info.max_mtu != UINT16_MAX &&
3357             port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3358                 eth_overhead = port->dev_info.max_rx_pktlen -
3359                                 port->dev_info.max_mtu;
3360         else
3361                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3362
3363         rx_offloads = port->dev_conf.rxmode.offloads;
3364
3365         /* Default config value is 0 to use PMD specific overhead */
3366         if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3367                 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3368
3369         if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3370                 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3371                 on = false;
3372         } else {
3373                 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3374                         printf("Frame size (%u) is not supported by port %u\n",
3375                                 port->dev_conf.rxmode.max_rx_pkt_len,
3376                                 portid);
3377                         return -1;
3378                 }
3379                 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3380                 on = true;
3381         }
3382
3383         if (rx_offloads != port->dev_conf.rxmode.offloads) {
3384                 uint16_t qid;
3385
3386                 port->dev_conf.rxmode.offloads = rx_offloads;
3387
3388                 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3389                 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3390                         if (on)
3391                                 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3392                         else
3393                                 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3394                 }
3395         }
3396
3397         /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3398          * if unset do it here
3399          */
3400         if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3401                 ret = rte_eth_dev_set_mtu(portid,
3402                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3403                 if (ret)
3404                         printf("Failed to set MTU to %u for port %u\n",
3405                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3406                                 portid);
3407         }
3408
3409         return 0;
3410 }
3411
3412 void
3413 init_port_config(void)
3414 {
3415         portid_t pid;
3416         struct rte_port *port;
3417         int ret;
3418
3419         RTE_ETH_FOREACH_DEV(pid) {
3420                 port = &ports[pid];
3421                 port->dev_conf.fdir_conf = fdir_conf;
3422
3423                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3424                 if (ret != 0)
3425                         return;
3426
3427                 if (nb_rxq > 1) {
3428                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3429                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3430                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3431                 } else {
3432                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3433                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3434                 }
3435
3436                 if (port->dcb_flag == 0) {
3437                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3438                                 port->dev_conf.rxmode.mq_mode =
3439                                         (enum rte_eth_rx_mq_mode)
3440                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3441                         else
3442                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3443                 }
3444
3445                 rxtx_port_config(port);
3446
3447                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3448                 if (ret != 0)
3449                         return;
3450
3451 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3452                 rte_pmd_ixgbe_bypass_init(pid);
3453 #endif
3454
3455                 if (lsc_interrupt &&
3456                     (rte_eth_devices[pid].data->dev_flags &
3457                      RTE_ETH_DEV_INTR_LSC))
3458                         port->dev_conf.intr_conf.lsc = 1;
3459                 if (rmv_interrupt &&
3460                     (rte_eth_devices[pid].data->dev_flags &
3461                      RTE_ETH_DEV_INTR_RMV))
3462                         port->dev_conf.intr_conf.rmv = 1;
3463         }
3464 }
3465
3466 void set_port_slave_flag(portid_t slave_pid)
3467 {
3468         struct rte_port *port;
3469
3470         port = &ports[slave_pid];
3471         port->slave_flag = 1;
3472 }
3473
3474 void clear_port_slave_flag(portid_t slave_pid)
3475 {
3476         struct rte_port *port;
3477
3478         port = &ports[slave_pid];
3479         port->slave_flag = 0;
3480 }
3481
3482 uint8_t port_is_bonding_slave(portid_t slave_pid)
3483 {
3484         struct rte_port *port;
3485
3486         port = &ports[slave_pid];
3487         if ((rte_eth_devices[slave_pid].data->dev_flags &
3488             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3489                 return 1;
3490         return 0;
3491 }
3492
3493 const uint16_t vlan_tags[] = {
3494                 0,  1,  2,  3,  4,  5,  6,  7,
3495                 8,  9, 10, 11,  12, 13, 14, 15,
3496                 16, 17, 18, 19, 20, 21, 22, 23,
3497                 24, 25, 26, 27, 28, 29, 30, 31
3498 };
3499
3500 static  int
3501 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3502                  enum dcb_mode_enable dcb_mode,
3503                  enum rte_eth_nb_tcs num_tcs,
3504                  uint8_t pfc_en)
3505 {
3506         uint8_t i;
3507         int32_t rc;
3508         struct rte_eth_rss_conf rss_conf;
3509
3510         /*
3511          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3512          * given above, and the number of traffic classes available for use.
3513          */
3514         if (dcb_mode == DCB_VT_ENABLED) {
3515                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3516                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3517                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3518                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3519
3520                 /* VMDQ+DCB RX and TX configurations */
3521                 vmdq_rx_conf->enable_default_pool = 0;
3522                 vmdq_rx_conf->default_pool = 0;
3523                 vmdq_rx_conf->nb_queue_pools =
3524                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3525                 vmdq_tx_conf->nb_queue_pools =
3526                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3527
3528                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3529                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3530                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3531                         vmdq_rx_conf->pool_map[i].pools =
3532                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3533                 }
3534                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3535                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3536                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3537                 }
3538
3539                 /* set DCB mode of RX and TX of multiple queues */
3540                 eth_conf->rxmode.mq_mode =
3541                                 (enum rte_eth_rx_mq_mode)
3542                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3543                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3544         } else {
3545                 struct rte_eth_dcb_rx_conf *rx_conf =
3546                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3547                 struct rte_eth_dcb_tx_conf *tx_conf =
3548                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3549
3550                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3551
3552                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3553                 if (rc != 0)
3554                         return rc;
3555
3556                 rx_conf->nb_tcs = num_tcs;
3557                 tx_conf->nb_tcs = num_tcs;
3558
3559                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3560                         rx_conf->dcb_tc[i] = i % num_tcs;
3561                         tx_conf->dcb_tc[i] = i % num_tcs;
3562                 }
3563
3564                 eth_conf->rxmode.mq_mode =
3565                                 (enum rte_eth_rx_mq_mode)
3566                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3567                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3568                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3569         }
3570
3571         if (pfc_en)
3572                 eth_conf->dcb_capability_en =
3573                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3574         else
3575                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3576
3577         return 0;
3578 }
3579
3580 int
3581 init_port_dcb_config(portid_t pid,
3582                      enum dcb_mode_enable dcb_mode,
3583                      enum rte_eth_nb_tcs num_tcs,
3584                      uint8_t pfc_en)
3585 {
3586         struct rte_eth_conf port_conf;
3587         struct rte_port *rte_port;
3588         int retval;
3589         uint16_t i;
3590
3591         rte_port = &ports[pid];
3592
3593         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3594
3595         port_conf.rxmode = rte_port->dev_conf.rxmode;
3596         port_conf.txmode = rte_port->dev_conf.txmode;
3597
3598         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3599         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3600         if (retval < 0)
3601                 return retval;
3602         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3603
3604         /* re-configure the device . */
3605         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3606         if (retval < 0)
3607                 return retval;
3608
3609         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3610         if (retval != 0)
3611                 return retval;
3612
3613         /* If dev_info.vmdq_pool_base is greater than 0,
3614          * the queue id of vmdq pools is started after pf queues.
3615          */
3616         if (dcb_mode == DCB_VT_ENABLED &&
3617             rte_port->dev_info.vmdq_pool_base > 0) {
3618                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3619                         " for port %d.", pid);
3620                 return -1;
3621         }
3622
3623         /* Assume the ports in testpmd have the same dcb capability
3624          * and has the same number of rxq and txq in dcb mode
3625          */
3626         if (dcb_mode == DCB_VT_ENABLED) {
3627                 if (rte_port->dev_info.max_vfs > 0) {
3628                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3629                         nb_txq = rte_port->dev_info.nb_tx_queues;
3630                 } else {
3631                         nb_rxq = rte_port->dev_info.max_rx_queues;
3632                         nb_txq = rte_port->dev_info.max_tx_queues;
3633                 }
3634         } else {
3635                 /*if vt is disabled, use all pf queues */
3636                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3637                         nb_rxq = rte_port->dev_info.max_rx_queues;
3638                         nb_txq = rte_port->dev_info.max_tx_queues;
3639                 } else {
3640                         nb_rxq = (queueid_t)num_tcs;
3641                         nb_txq = (queueid_t)num_tcs;
3642
3643                 }
3644         }
3645         rx_free_thresh = 64;
3646
3647         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3648
3649         rxtx_port_config(rte_port);
3650         /* VLAN filter */
3651         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3652         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3653                 rx_vft_set(pid, vlan_tags[i], 1);
3654
3655         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3656         if (retval != 0)
3657                 return retval;
3658
3659         rte_port->dcb_flag = 1;
3660
3661         /* Enter DCB configuration status */
3662         dcb_config = 1;
3663
3664         return 0;
3665 }
3666
3667 static void
3668 init_port(void)
3669 {
3670         int i;
3671
3672         /* Configuration of Ethernet ports. */
3673         ports = rte_zmalloc("testpmd: ports",
3674                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3675                             RTE_CACHE_LINE_SIZE);
3676         if (ports == NULL) {
3677                 rte_exit(EXIT_FAILURE,
3678                                 "rte_zmalloc(%d struct rte_port) failed\n",
3679                                 RTE_MAX_ETHPORTS);
3680         }
3681         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3682                 LIST_INIT(&ports[i].flow_tunnel_list);
3683         /* Initialize ports NUMA structures */
3684         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3685         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3686         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3687 }
3688
3689 static void
3690 force_quit(void)
3691 {
3692         pmd_test_exit();
3693         prompt_exit();
3694 }
3695
3696 static void
3697 print_stats(void)
3698 {
3699         uint8_t i;
3700         const char clr[] = { 27, '[', '2', 'J', '\0' };
3701         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3702
3703         /* Clear screen and move to top left */
3704         printf("%s%s", clr, top_left);
3705
3706         printf("\nPort statistics ====================================");
3707         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3708                 nic_stats_display(fwd_ports_ids[i]);
3709
3710         fflush(stdout);
3711 }
3712
3713 static void
3714 signal_handler(int signum)
3715 {
3716         if (signum == SIGINT || signum == SIGTERM) {
3717                 printf("\nSignal %d received, preparing to exit...\n",
3718                                 signum);
3719 #ifdef RTE_LIB_PDUMP
3720                 /* uninitialize packet capture framework */
3721                 rte_pdump_uninit();
3722 #endif
3723 #ifdef RTE_LIB_LATENCYSTATS
3724                 if (latencystats_enabled != 0)
3725                         rte_latencystats_uninit();
3726 #endif
3727                 force_quit();
3728                 /* Set flag to indicate the force termination. */
3729                 f_quit = 1;
3730                 /* exit with the expected status */
3731 #ifndef RTE_EXEC_ENV_WINDOWS
3732                 signal(signum, SIG_DFL);
3733                 kill(getpid(), signum);
3734 #endif
3735         }
3736 }
3737
3738 int
3739 main(int argc, char** argv)
3740 {
3741         int diag;
3742         portid_t port_id;
3743         uint16_t count;
3744         int ret;
3745
3746         signal(SIGINT, signal_handler);
3747         signal(SIGTERM, signal_handler);
3748
3749         testpmd_logtype = rte_log_register("testpmd");
3750         if (testpmd_logtype < 0)
3751                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3752         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3753
3754         diag = rte_eal_init(argc, argv);
3755         if (diag < 0)
3756                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3757                          rte_strerror(rte_errno));
3758
3759         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3760                 rte_exit(EXIT_FAILURE,
3761                          "Secondary process type not supported.\n");
3762
3763         ret = register_eth_event_callback();
3764         if (ret != 0)
3765                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3766
3767 #ifdef RTE_LIB_PDUMP
3768         /* initialize packet capture framework */
3769         rte_pdump_init();
3770 #endif
3771
3772         count = 0;
3773         RTE_ETH_FOREACH_DEV(port_id) {
3774                 ports_ids[count] = port_id;
3775                 count++;
3776         }
3777         nb_ports = (portid_t) count;
3778         if (nb_ports == 0)
3779                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3780
3781         /* allocate port structures, and init them */
3782         init_port();
3783
3784         set_def_fwd_config();
3785         if (nb_lcores == 0)
3786                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3787                          "Check the core mask argument\n");
3788
3789         /* Bitrate/latency stats disabled by default */
3790 #ifdef RTE_LIB_BITRATESTATS
3791         bitrate_enabled = 0;
3792 #endif
3793 #ifdef RTE_LIB_LATENCYSTATS
3794         latencystats_enabled = 0;
3795 #endif
3796
3797         /* on FreeBSD, mlockall() is disabled by default */
3798 #ifdef RTE_EXEC_ENV_FREEBSD
3799         do_mlockall = 0;
3800 #else
3801         do_mlockall = 1;
3802 #endif
3803
3804         argc -= diag;
3805         argv += diag;
3806         if (argc > 1)
3807                 launch_args_parse(argc, argv);
3808
3809 #ifndef RTE_EXEC_ENV_WINDOWS
3810         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3811                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3812                         strerror(errno));
3813         }
3814 #endif
3815
3816         if (tx_first && interactive)
3817                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3818                                 "interactive mode.\n");
3819
3820         if (tx_first && lsc_interrupt) {
3821                 printf("Warning: lsc_interrupt needs to be off when "
3822                                 " using tx_first. Disabling.\n");
3823                 lsc_interrupt = 0;
3824         }
3825
3826         if (!nb_rxq && !nb_txq)
3827                 printf("Warning: Either rx or tx queues should be non-zero\n");
3828
3829         if (nb_rxq > 1 && nb_rxq > nb_txq)
3830                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3831                        "but nb_txq=%d will prevent to fully test it.\n",
3832                        nb_rxq, nb_txq);
3833
3834         init_config();
3835
3836         if (hot_plug) {
3837                 ret = rte_dev_hotplug_handle_enable();
3838                 if (ret) {
3839                         RTE_LOG(ERR, EAL,
3840                                 "fail to enable hotplug handling.");
3841                         return -1;
3842                 }
3843
3844                 ret = rte_dev_event_monitor_start();
3845                 if (ret) {
3846                         RTE_LOG(ERR, EAL,
3847                                 "fail to start device event monitoring.");
3848                         return -1;
3849                 }
3850
3851                 ret = rte_dev_event_callback_register(NULL,
3852                         dev_event_callback, NULL);
3853                 if (ret) {
3854                         RTE_LOG(ERR, EAL,
3855                                 "fail  to register device event callback\n");
3856                         return -1;
3857                 }
3858         }
3859
3860         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3861                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3862
3863         /* set all ports to promiscuous mode by default */
3864         RTE_ETH_FOREACH_DEV(port_id) {
3865                 ret = rte_eth_promiscuous_enable(port_id);
3866                 if (ret != 0)
3867                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3868                                 port_id, rte_strerror(-ret));
3869         }
3870
3871         /* Init metrics library */
3872         rte_metrics_init(rte_socket_id());
3873
3874 #ifdef RTE_LIB_LATENCYSTATS
3875         if (latencystats_enabled != 0) {
3876                 int ret = rte_latencystats_init(1, NULL);
3877                 if (ret)
3878                         printf("Warning: latencystats init()"
3879                                 " returned error %d\n", ret);
3880                 printf("Latencystats running on lcore %d\n",
3881                         latencystats_lcore_id);
3882         }
3883 #endif
3884
3885         /* Setup bitrate stats */
3886 #ifdef RTE_LIB_BITRATESTATS
3887         if (bitrate_enabled != 0) {
3888                 bitrate_data = rte_stats_bitrate_create();
3889                 if (bitrate_data == NULL)
3890                         rte_exit(EXIT_FAILURE,
3891                                 "Could not allocate bitrate data.\n");
3892                 rte_stats_bitrate_reg(bitrate_data);
3893         }
3894 #endif
3895
3896 #ifdef RTE_LIB_CMDLINE
3897         if (strlen(cmdline_filename) != 0)
3898                 cmdline_read_from_file(cmdline_filename);
3899
3900         if (interactive == 1) {
3901                 if (auto_start) {
3902                         printf("Start automatic packet forwarding\n");
3903                         start_packet_forwarding(0);
3904                 }
3905                 prompt();
3906                 pmd_test_exit();
3907         } else
3908 #endif
3909         {
3910                 char c;
3911                 int rc;
3912
3913                 f_quit = 0;
3914
3915                 printf("No commandline core given, start packet forwarding\n");
3916                 start_packet_forwarding(tx_first);
3917                 if (stats_period != 0) {
3918                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3919                         uint64_t timer_period;
3920
3921                         /* Convert to number of cycles */
3922                         timer_period = stats_period * rte_get_timer_hz();
3923
3924                         while (f_quit == 0) {
3925                                 cur_time = rte_get_timer_cycles();
3926                                 diff_time += cur_time - prev_time;
3927
3928                                 if (diff_time >= timer_period) {
3929                                         print_stats();
3930                                         /* Reset the timer */
3931                                         diff_time = 0;
3932                                 }
3933                                 /* Sleep to avoid unnecessary checks */
3934                                 prev_time = cur_time;
3935                                 rte_delay_us_sleep(US_PER_S);
3936                         }
3937                 }
3938
3939                 printf("Press enter to exit\n");
3940                 rc = read(0, &c, 1);
3941                 pmd_test_exit();
3942                 if (rc < 0)
3943                         return 1;
3944         }
3945
3946         ret = rte_eal_cleanup();
3947         if (ret != 0)
3948                 rte_exit(EXIT_FAILURE,
3949                          "EAL cleanup failed: %s\n", strerror(-ret));
3950
3951         return EXIT_SUCCESS;
3952 }