app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #ifndef RTE_EXEC_ENV_WINDOWS
  13 #include <sys/mman.h>
  14 #endif
  15 #include <sys/types.h>
  16 #include <errno.h>
  17 #include <stdbool.h>
  18
  19 #include <sys/queue.h>
  20 #include <sys/stat.h>
  21
  22 #include <stdint.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_byteorder.h>
  29 #include <rte_log.h>
  30 #include <rte_debug.h>
  31 #include <rte_cycles.h>
  32 #include <rte_memory.h>
  33 #include <rte_memcpy.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_alarm.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_atomic.h>
  40 #include <rte_branch_prediction.h>
  41 #include <rte_mempool.h>
  42 #include <rte_malloc.h>
  43 #include <rte_mbuf.h>
  44 #include <rte_mbuf_pool_ops.h>
  45 #include <rte_interrupts.h>
  46 #include <rte_pci.h>
  47 #include <rte_ether.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_dev.h>
  50 #include <rte_string_fns.h>
  51 #ifdef RTE_NET_IXGBE
  52 #include <rte_pmd_ixgbe.h>
  53 #endif
  54 #ifdef RTE_LIB_PDUMP
  55 #include <rte_pdump.h>
  56 #endif
  57 #include <rte_flow.h>
  58 #include <rte_metrics.h>
  59 #ifdef RTE_LIB_BITRATESTATS
  60 #include <rte_bitrate.h>
  61 #endif
  62 #ifdef RTE_LIB_LATENCYSTATS
  63 #include <rte_latencystats.h>
  64 #endif
  65 #ifdef RTE_EXEC_ENV_WINDOWS
  66 #include <process.h>
  67 #endif
  68
  69 #include "testpmd.h"
  70
  71 #ifndef MAP_HUGETLB
  72 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  73 #define HUGE_FLAG (0x40000)
  74 #else
  75 #define HUGE_FLAG MAP_HUGETLB
  76 #endif
  77
  78 #ifndef MAP_HUGE_SHIFT
  79 /* older kernels (or FreeBSD) will not have this define */
  80 #define HUGE_SHIFT (26)
  81 #else
  82 #define HUGE_SHIFT MAP_HUGE_SHIFT
  83 #endif
  84
  85 #define EXTMEM_HEAP_NAME "extmem"
  86 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  87
  88 uint16_t verbose_level = 0; /**< Silent by default. */
  89 int testpmd_logtype; /**< Log type for testpmd logs */
  90
  91 /* use main core for command line ? */
  92 uint8_t interactive = 0;
  93 uint8_t auto_start = 0;
  94 uint8_t tx_first;
  95 char cmdline_filename[PATH_MAX] = {0};
  96
  97 /*
  98  * NUMA support configuration.
  99  * When set, the NUMA support attempts to dispatch the allocation of the
 100  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 101  * probed ports among the CPU sockets 0 and 1.
 102  * Otherwise, all memory is allocated from CPU socket 0.
 103  */
 104 uint8_t numa_support = 1; /**< numa enabled by default */
 105
 106 /*
 107  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 108  * not configured.
 109  */
 110 uint8_t socket_num = UMA_NO_CONFIG;
 111
 112 /*
 113  * Select mempool allocation type:
 114  * - native: use regular DPDK memory
 115  * - anon: use regular DPDK memory to create mempool, but populate using
 116  *         anonymous memory (may not be IOVA-contiguous)
 117  * - xmem: use externally allocated hugepage memory
 118  */
 119 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 120
 121 /*
 122  * Store specified sockets on which memory pool to be used by ports
 123  * is allocated.
 124  */
 125 uint8_t port_numa[RTE_MAX_ETHPORTS];
 126
 127 /*
 128  * Store specified sockets on which RX ring to be used by ports
 129  * is allocated.
 130  */
 131 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 132
 133 /*
 134  * Store specified sockets on which TX ring to be used by ports
 135  * is allocated.
 136  */
 137 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 138
 139 /*
 140  * Record the Ethernet address of peer target ports to which packets are
 141  * forwarded.
 142  * Must be instantiated with the ethernet addresses of peer traffic generator
 143  * ports.
 144  */
 145 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 146 portid_t nb_peer_eth_addrs = 0;
 147
 148 /*
 149  * Probed Target Environment.
 150  */
 151 struct rte_port *ports;        /**< For all probed ethernet ports. */
 152 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 153 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 154 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 155
 156 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 157
 158 /*
 159  * Test Forwarding Configuration.
 160  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 161  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 162  */
 163 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 164 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 165 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 166 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 167
 168 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 169 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 170
 171 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 172 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 173
 174 /*
 175  * Forwarding engines.
 176  */
 177 struct fwd_engine * fwd_engines[] = {
 178         &io_fwd_engine,
 179         &mac_fwd_engine,
 180         &mac_swap_engine,
 181         &flow_gen_engine,
 182         &rx_only_engine,
 183         &tx_only_engine,
 184         &csum_fwd_engine,
 185         &icmp_echo_engine,
 186         &noisy_vnf_engine,
 187         &five_tuple_swap_fwd_engine,
 188 #ifdef RTE_LIBRTE_IEEE1588
 189         &ieee1588_fwd_engine,
 190 #endif
 191         NULL,
 192 };
 193
 194 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 195 uint16_t mempool_flags;
 196
 197 struct fwd_config cur_fwd_config;
 198 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 199 uint32_t retry_enabled;
 200 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 201 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 202
 203 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 204 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 205         DEFAULT_MBUF_DATA_SIZE
 206 }; /**< Mbuf data space size. */
 207 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 208                                       * specified on command-line. */
 209 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 210
 211 /*
 212  * In container, it cannot terminate the process which running with 'stats-period'
 213  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 214  */
 215 uint8_t f_quit;
 216
 217 /*
 218  * Configuration of packet segments used to scatter received packets
 219  * if some of split features is configured.
 220  */
 221 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 222 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 223 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 224 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 225
 226 /*
 227  * Configuration of packet segments used by the "txonly" processing engine.
 228  */
 229 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 230 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 231         TXONLY_DEF_PACKET_LEN,
 232 };
 233 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 234
 235 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 236 /**< Split policy for packets to TX. */
 237
 238 uint8_t txonly_multi_flow;
 239 /**< Whether multiple flows are generated in TXONLY mode. */
 240
 241 uint32_t tx_pkt_times_inter;
 242 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 243
 244 uint32_t tx_pkt_times_intra;
 245 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 246
 247 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 248 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 249 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 250
 251 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 252 uint8_t dcb_config = 0;
 253
 254 /*
 255  * Configurable number of RX/TX queues.
 256  */
 257 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 258 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 259 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 260
 261 /*
 262  * Configurable number of RX/TX ring descriptors.
 263  * Defaults are supplied by drivers via ethdev.
 264  */
 265 #define RTE_TEST_RX_DESC_DEFAULT 0
 266 #define RTE_TEST_TX_DESC_DEFAULT 0
 267 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 268 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 269
 270 #define RTE_PMD_PARAM_UNSET -1
 271 /*
 272  * Configurable values of RX and TX ring threshold registers.
 273  */
 274
 275 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 276 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 277 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 278
 279 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 280 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 281 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of RX free threshold.
 285  */
 286 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 287
 288 /*
 289  * Configurable value of RX drop enable.
 290  */
 291 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 292
 293 /*
 294  * Configurable value of TX free threshold.
 295  */
 296 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 297
 298 /*
 299  * Configurable value of TX RS bit threshold.
 300  */
 301 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 302
 303 /*
 304  * Configurable value of buffered packets before sending.
 305  */
 306 uint16_t noisy_tx_sw_bufsz;
 307
 308 /*
 309  * Configurable value of packet buffer timeout.
 310  */
 311 uint16_t noisy_tx_sw_buf_flush_time;
 312
 313 /*
 314  * Configurable value for size of VNF internal memory area
 315  * used for simulating noisy neighbour behaviour
 316  */
 317 uint64_t noisy_lkup_mem_sz;
 318
 319 /*
 320  * Configurable value of number of random writes done in
 321  * VNF simulation memory area.
 322  */
 323 uint64_t noisy_lkup_num_writes;
 324
 325 /*
 326  * Configurable value of number of random reads done in
 327  * VNF simulation memory area.
 328  */
 329 uint64_t noisy_lkup_num_reads;
 330
 331 /*
 332  * Configurable value of number of random reads/writes done in
 333  * VNF simulation memory area.
 334  */
 335 uint64_t noisy_lkup_num_reads_writes;
 336
 337 /*
 338  * Receive Side Scaling (RSS) configuration.
 339  */
 340 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 341
 342 /*
 343  * Port topology configuration
 344  */
 345 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 346
 347 /*
 348  * Avoids to flush all the RX streams before starts forwarding.
 349  */
 350 uint8_t no_flush_rx = 0; /* flush by default */
 351
 352 /*
 353  * Flow API isolated mode.
 354  */
 355 uint8_t flow_isolate_all;
 356
 357 /*
 358  * Avoids to check link status when starting/stopping a port.
 359  */
 360 uint8_t no_link_check = 0; /* check by default */
 361
 362 /*
 363  * Don't automatically start all ports in interactive mode.
 364  */
 365 uint8_t no_device_start = 0;
 366
 367 /*
 368  * Enable link status change notification
 369  */
 370 uint8_t lsc_interrupt = 1; /* enabled by default */
 371
 372 /*
 373  * Enable device removal notification.
 374  */
 375 uint8_t rmv_interrupt = 1; /* enabled by default */
 376
 377 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 378
 379 /* After attach, port setup is called on event or by iterator */
 380 bool setup_on_probe_event = true;
 381
 382 /* Clear ptypes on port initialization. */
 383 uint8_t clear_ptypes = true;
 384
 385 /* Hairpin ports configuration mode. */
 386 uint16_t hairpin_mode;
 387
 388 /* Pretty printing of ethdev events */
 389 static const char * const eth_event_desc[] = {
 390         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 391         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 392         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 393         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 394         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 395         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 396         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 397         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 398         [RTE_ETH_EVENT_NEW] = "device probed",
 399         [RTE_ETH_EVENT_DESTROY] = "device released",
 400         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 401         [RTE_ETH_EVENT_MAX] = NULL,
 402 };
 403
 404 /*
 405  * Display or mask ether events
 406  * Default to all events except VF_MBOX
 407  */
 408 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 413                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 414                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 415                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 416 /*
 417  * Decide if all memory are locked for performance.
 418  */
 419 int do_mlockall = 0;
 420
 421 /*
 422  * NIC bypass mode configuration options.
 423  */
 424
 425 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 426 /* The NIC bypass watchdog timeout. */
 427 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 428 #endif
 429
 430
 431 #ifdef RTE_LIB_LATENCYSTATS
 432
 433 /*
 434  * Set when latency stats is enabled in the commandline
 435  */
 436 uint8_t latencystats_enabled;
 437
 438 /*
 439  * Lcore ID to serive latency statistics.
 440  */
 441 lcoreid_t latencystats_lcore_id = -1;
 442
 443 #endif
 444
 445 /*
 446  * Ethernet device configuration.
 447  */
 448 struct rte_eth_rxmode rx_mode = {
 449         /* Default maximum frame length.
 450          * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
 451          * in init_config().
 452          */
 453         .max_rx_pkt_len = 0,
 454 };
 455
 456 struct rte_eth_txmode tx_mode = {
 457         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 458 };
 459
 460 struct rte_fdir_conf fdir_conf = {
 461         .mode = RTE_FDIR_MODE_NONE,
 462         .pballoc = RTE_FDIR_PBALLOC_64K,
 463         .status = RTE_FDIR_REPORT_STATUS,
 464         .mask = {
 465                 .vlan_tci_mask = 0xFFEF,
 466                 .ipv4_mask     = {
 467                         .src_ip = 0xFFFFFFFF,
 468                         .dst_ip = 0xFFFFFFFF,
 469                 },
 470                 .ipv6_mask     = {
 471                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 472                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 473                 },
 474                 .src_port_mask = 0xFFFF,
 475                 .dst_port_mask = 0xFFFF,
 476                 .mac_addr_byte_mask = 0xFF,
 477                 .tunnel_type_mask = 1,
 478                 .tunnel_id_mask = 0xFFFFFFFF,
 479         },
 480         .drop_queue = 127,
 481 };
 482
 483 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 484
 485 /*
 486  * Display zero values by default for xstats
 487  */
 488 uint8_t xstats_hide_zero;
 489
 490 /*
 491  * Measure of CPU cycles disabled by default
 492  */
 493 uint8_t record_core_cycles;
 494
 495 /*
 496  * Display of RX and TX bursts disabled by default
 497  */
 498 uint8_t record_burst_stats;
 499
 500 unsigned int num_sockets = 0;
 501 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 502
 503 #ifdef RTE_LIB_BITRATESTATS
 504 /* Bitrate statistics */
 505 struct rte_stats_bitrates *bitrate_data;
 506 lcoreid_t bitrate_lcore_id;
 507 uint8_t bitrate_enabled;
 508 #endif
 509
 510 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 511 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 512
 513 /*
 514  * hexadecimal bitmask of RX mq mode can be enabled.
 515  */
 516 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 517
 518 /*
 519  * Used to set forced link speed
 520  */
 521 uint32_t eth_link_speed;
 522
 523 /* Forward function declarations */
 524 static void setup_attached_port(portid_t pi);
 525 static void check_all_ports_link_status(uint32_t port_mask);
 526 static int eth_event_callback(portid_t port_id,
 527                               enum rte_eth_event_type type,
 528                               void *param, void *ret_param);
 529 static void dev_event_callback(const char *device_name,
 530                                 enum rte_dev_event_type type,
 531                                 void *param);
 532
 533 /*
 534  * Check if all the ports are started.
 535  * If yes, return positive value. If not, return zero.
 536  */
 537 static int all_ports_started(void);
 538
 539 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 540 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 541
 542 /* Holds the registered mbuf dynamic flags names. */
 543 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 544
 545 /*
 546  * Helper function to check if socket is already discovered.
 547  * If yes, return positive value. If not, return zero.
 548  */
 549 int
 550 new_socket_id(unsigned int socket_id)
 551 {
 552         unsigned int i;
 553
 554         for (i = 0; i < num_sockets; i++) {
 555                 if (socket_ids[i] == socket_id)
 556                         return 0;
 557         }
 558         return 1;
 559 }
 560
 561 /*
 562  * Setup default configuration.
 563  */
 564 static void
 565 set_default_fwd_lcores_config(void)
 566 {
 567         unsigned int i;
 568         unsigned int nb_lc;
 569         unsigned int sock_num;
 570
 571         nb_lc = 0;
 572         for (i = 0; i < RTE_MAX_LCORE; i++) {
 573                 if (!rte_lcore_is_enabled(i))
 574                         continue;
 575                 sock_num = rte_lcore_to_socket_id(i);
 576                 if (new_socket_id(sock_num)) {
 577                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 578                                 rte_exit(EXIT_FAILURE,
 579                                          "Total sockets greater than %u\n",
 580                                          RTE_MAX_NUMA_NODES);
 581                         }
 582                         socket_ids[num_sockets++] = sock_num;
 583                 }
 584                 if (i == rte_get_main_lcore())
 585                         continue;
 586                 fwd_lcores_cpuids[nb_lc++] = i;
 587         }
 588         nb_lcores = (lcoreid_t) nb_lc;
 589         nb_cfg_lcores = nb_lcores;
 590         nb_fwd_lcores = 1;
 591 }
 592
 593 static void
 594 set_def_peer_eth_addrs(void)
 595 {
 596         portid_t i;
 597
 598         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 599                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 600                 peer_eth_addrs[i].addr_bytes[5] = i;
 601         }
 602 }
 603
 604 static void
 605 set_default_fwd_ports_config(void)
 606 {
 607         portid_t pt_id;
 608         int i = 0;
 609
 610         RTE_ETH_FOREACH_DEV(pt_id) {
 611                 fwd_ports_ids[i++] = pt_id;
 612
 613                 /* Update sockets info according to the attached device */
 614                 int socket_id = rte_eth_dev_socket_id(pt_id);
 615                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 616                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 617                                 rte_exit(EXIT_FAILURE,
 618                                          "Total sockets greater than %u\n",
 619                                          RTE_MAX_NUMA_NODES);
 620                         }
 621                         socket_ids[num_sockets++] = socket_id;
 622                 }
 623         }
 624
 625         nb_cfg_ports = nb_ports;
 626         nb_fwd_ports = nb_ports;
 627 }
 628
 629 void
 630 set_def_fwd_config(void)
 631 {
 632         set_default_fwd_lcores_config();
 633         set_def_peer_eth_addrs();
 634         set_default_fwd_ports_config();
 635 }
 636
 637 #ifndef RTE_EXEC_ENV_WINDOWS
 638 /* extremely pessimistic estimation of memory required to create a mempool */
 639 static int
 640 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 641 {
 642         unsigned int n_pages, mbuf_per_pg, leftover;
 643         uint64_t total_mem, mbuf_mem, obj_sz;
 644
 645         /* there is no good way to predict how much space the mempool will
 646          * occupy because it will allocate chunks on the fly, and some of those
 647          * will come from default DPDK memory while some will come from our
 648          * external memory, so just assume 128MB will be enough for everyone.
 649          */
 650         uint64_t hdr_mem = 128 << 20;
 651
 652         /* account for possible non-contiguousness */
 653         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 654         if (obj_sz > pgsz) {
 655                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 656                 return -1;
 657         }
 658
 659         mbuf_per_pg = pgsz / obj_sz;
 660         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 661         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 662
 663         mbuf_mem = n_pages * pgsz;
 664
 665         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 666
 667         if (total_mem > SIZE_MAX) {
 668                 TESTPMD_LOG(ERR, "Memory size too big\n");
 669                 return -1;
 670         }
 671         *out = (size_t)total_mem;
 672
 673         return 0;
 674 }
 675
 676 static int
 677 pagesz_flags(uint64_t page_sz)
 678 {
 679         /* as per mmap() manpage, all page sizes are log2 of page size
 680          * shifted by MAP_HUGE_SHIFT
 681          */
 682         int log2 = rte_log2_u64(page_sz);
 683
 684         return (log2 << HUGE_SHIFT);
 685 }
 686
 687 static void *
 688 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 689 {
 690         void *addr;
 691         int flags;
 692
 693         /* allocate anonymous hugepages */
 694         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 695         if (huge)
 696                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 697
 698         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 699         if (addr == MAP_FAILED)
 700                 return NULL;
 701
 702         return addr;
 703 }
 704
 705 struct extmem_param {
 706         void *addr;
 707         size_t len;
 708         size_t pgsz;
 709         rte_iova_t *iova_table;
 710         unsigned int iova_table_len;
 711 };
 712
 713 static int
 714 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 715                 bool huge)
 716 {
 717         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 718                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 719         unsigned int cur_page, n_pages, pgsz_idx;
 720         size_t mem_sz, cur_pgsz;
 721         rte_iova_t *iovas = NULL;
 722         void *addr;
 723         int ret;
 724
 725         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 726                 /* skip anything that is too big */
 727                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 728                         continue;
 729
 730                 cur_pgsz = pgsizes[pgsz_idx];
 731
 732                 /* if we were told not to allocate hugepages, override */
 733                 if (!huge)
 734                         cur_pgsz = sysconf(_SC_PAGESIZE);
 735
 736                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 737                 if (ret < 0) {
 738                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 739                         return -1;
 740                 }
 741
 742                 /* allocate our memory */
 743                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 744
 745                 /* if we couldn't allocate memory with a specified page size,
 746                  * that doesn't mean we can't do it with other page sizes, so
 747                  * try another one.
 748                  */
 749                 if (addr == NULL)
 750                         continue;
 751
 752                 /* store IOVA addresses for every page in this memory area */
 753                 n_pages = mem_sz / cur_pgsz;
 754
 755                 iovas = malloc(sizeof(*iovas) * n_pages);
 756
 757                 if (iovas == NULL) {
 758                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 759                         goto fail;
 760                 }
 761                 /* lock memory if it's not huge pages */
 762                 if (!huge)
 763                         mlock(addr, mem_sz);
 764
 765                 /* populate IOVA addresses */
 766                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 767                         rte_iova_t iova;
 768                         size_t offset;
 769                         void *cur;
 770
 771                         offset = cur_pgsz * cur_page;
 772                         cur = RTE_PTR_ADD(addr, offset);
 773
 774                         /* touch the page before getting its IOVA */
 775                         *(volatile char *)cur = 0;
 776
 777                         iova = rte_mem_virt2iova(cur);
 778
 779                         iovas[cur_page] = iova;
 780                 }
 781
 782                 break;
 783         }
 784         /* if we couldn't allocate anything */
 785         if (iovas == NULL)
 786                 return -1;
 787
 788         param->addr = addr;
 789         param->len = mem_sz;
 790         param->pgsz = cur_pgsz;
 791         param->iova_table = iovas;
 792         param->iova_table_len = n_pages;
 793
 794         return 0;
 795 fail:
 796         if (iovas)
 797                 free(iovas);
 798         if (addr)
 799                 munmap(addr, mem_sz);
 800
 801         return -1;
 802 }
 803
 804 static int
 805 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 806 {
 807         struct extmem_param param;
 808         int socket_id, ret;
 809
 810         memset(&param, 0, sizeof(param));
 811
 812         /* check if our heap exists */
 813         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 814         if (socket_id < 0) {
 815                 /* create our heap */
 816                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 817                 if (ret < 0) {
 818                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 819                         return -1;
 820                 }
 821         }
 822
 823         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 824         if (ret < 0) {
 825                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 826                 return -1;
 827         }
 828
 829         /* we now have a valid memory area, so add it to heap */
 830         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 831                         param.addr, param.len, param.iova_table,
 832                         param.iova_table_len, param.pgsz);
 833
 834         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 835
 836         /* not needed any more */
 837         free(param.iova_table);
 838
 839         if (ret < 0) {
 840                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 841                 munmap(param.addr, param.len);
 842                 return -1;
 843         }
 844
 845         /* success */
 846
 847         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 848                         param.len >> 20);
 849
 850         return 0;
 851 }
 852 static void
 853 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 854              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 855 {
 856         uint16_t pid = 0;
 857         int ret;
 858
 859         RTE_ETH_FOREACH_DEV(pid) {
 860                 struct rte_eth_dev *dev =
 861                         &rte_eth_devices[pid];
 862
 863                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 864                                         memhdr->len);
 865                 if (ret) {
 866                         TESTPMD_LOG(DEBUG,
 867                                     "unable to DMA unmap addr 0x%p "
 868                                     "for device %s\n",
 869                                     memhdr->addr, dev->data->name);
 870                 }
 871         }
 872         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 873         if (ret) {
 874                 TESTPMD_LOG(DEBUG,
 875                             "unable to un-register addr 0x%p\n", memhdr->addr);
 876         }
 877 }
 878
 879 static void
 880 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 881            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 882 {
 883         uint16_t pid = 0;
 884         size_t page_size = sysconf(_SC_PAGESIZE);
 885         int ret;
 886
 887         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 888                                   page_size);
 889         if (ret) {
 890                 TESTPMD_LOG(DEBUG,
 891                             "unable to register addr 0x%p\n", memhdr->addr);
 892                 return;
 893         }
 894         RTE_ETH_FOREACH_DEV(pid) {
 895                 struct rte_eth_dev *dev =
 896                         &rte_eth_devices[pid];
 897
 898                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 899                                       memhdr->len);
 900                 if (ret) {
 901                         TESTPMD_LOG(DEBUG,
 902                                     "unable to DMA map addr 0x%p "
 903                                     "for device %s\n",
 904                                     memhdr->addr, dev->data->name);
 905                 }
 906         }
 907 }
 908 #endif
 909
 910 static unsigned int
 911 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 912             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 913 {
 914         struct rte_pktmbuf_extmem *xmem;
 915         unsigned int ext_num, zone_num, elt_num;
 916         uint16_t elt_size;
 917
 918         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 919         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 920         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 921
 922         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 923         if (xmem == NULL) {
 924                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 925                                  "external buffer descriptors\n");
 926                 *ext_mem = NULL;
 927                 return 0;
 928         }
 929         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 930                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 931                 const struct rte_memzone *mz;
 932                 char mz_name[RTE_MEMZONE_NAMESIZE];
 933                 int ret;
 934
 935                 ret = snprintf(mz_name, sizeof(mz_name),
 936                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 937                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 938                         errno = ENAMETOOLONG;
 939                         ext_num = 0;
 940                         break;
 941                 }
 942                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 943                                                  socket_id,
 944                                                  RTE_MEMZONE_IOVA_CONTIG |
 945                                                  RTE_MEMZONE_1GB |
 946                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 947                                                  EXTBUF_ZONE_SIZE);
 948                 if (mz == NULL) {
 949                         /*
 950                          * The caller exits on external buffer creation
 951                          * error, so there is no need to free memzones.
 952                          */
 953                         errno = ENOMEM;
 954                         ext_num = 0;
 955                         break;
 956                 }
 957                 xseg->buf_ptr = mz->addr;
 958                 xseg->buf_iova = mz->iova;
 959                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 960                 xseg->elt_size = elt_size;
 961         }
 962         if (ext_num == 0 && xmem != NULL) {
 963                 free(xmem);
 964                 xmem = NULL;
 965         }
 966         *ext_mem = xmem;
 967         return ext_num;
 968 }
 969
 970 /*
 971  * Configuration initialisation done once at init time.
 972  */
 973 static struct rte_mempool *
 974 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 975                  unsigned int socket_id, uint16_t size_idx)
 976 {
 977         char pool_name[RTE_MEMPOOL_NAMESIZE];
 978         struct rte_mempool *rte_mp = NULL;
 979 #ifndef RTE_EXEC_ENV_WINDOWS
 980         uint32_t mb_size;
 981
 982         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 983 #endif
 984         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 985
 986         TESTPMD_LOG(INFO,
 987                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 988                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 989
 990         switch (mp_alloc_type) {
 991         case MP_ALLOC_NATIVE:
 992                 {
 993                         /* wrapper to rte_mempool_create() */
 994                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 995                                         rte_mbuf_best_mempool_ops());
 996                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 997                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 998                         break;
 999                 }
1000 #ifndef RTE_EXEC_ENV_WINDOWS
1001         case MP_ALLOC_ANON:
1002                 {
1003                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1004                                 mb_size, (unsigned int) mb_mempool_cache,
1005                                 sizeof(struct rte_pktmbuf_pool_private),
1006                                 socket_id, mempool_flags);
1007                         if (rte_mp == NULL)
1008                                 goto err;
1009
1010                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1011                                 rte_mempool_free(rte_mp);
1012                                 rte_mp = NULL;
1013                                 goto err;
1014                         }
1015                         rte_pktmbuf_pool_init(rte_mp, NULL);
1016                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1017                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1018                         break;
1019                 }
1020         case MP_ALLOC_XMEM:
1021         case MP_ALLOC_XMEM_HUGE:
1022                 {
1023                         int heap_socket;
1024                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1025
1026                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1027                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1028
1029                         heap_socket =
1030                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1031                         if (heap_socket < 0)
1032                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1033
1034                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1035                                         rte_mbuf_best_mempool_ops());
1036                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1037                                         mb_mempool_cache, 0, mbuf_seg_size,
1038                                         heap_socket);
1039                         break;
1040                 }
1041 #endif
1042         case MP_ALLOC_XBUF:
1043                 {
1044                         struct rte_pktmbuf_extmem *ext_mem;
1045                         unsigned int ext_num;
1046
1047                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1048                                                socket_id, pool_name, &ext_mem);
1049                         if (ext_num == 0)
1050                                 rte_exit(EXIT_FAILURE,
1051                                          "Can't create pinned data buffers\n");
1052
1053                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1054                                         rte_mbuf_best_mempool_ops());
1055                         rte_mp = rte_pktmbuf_pool_create_extbuf
1056                                         (pool_name, nb_mbuf, mb_mempool_cache,
1057                                          0, mbuf_seg_size, socket_id,
1058                                          ext_mem, ext_num);
1059                         free(ext_mem);
1060                         break;
1061                 }
1062         default:
1063                 {
1064                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1065                 }
1066         }
1067
1068 #ifndef RTE_EXEC_ENV_WINDOWS
1069 err:
1070 #endif
1071         if (rte_mp == NULL) {
1072                 rte_exit(EXIT_FAILURE,
1073                         "Creation of mbuf pool for socket %u failed: %s\n",
1074                         socket_id, rte_strerror(rte_errno));
1075         } else if (verbose_level > 0) {
1076                 rte_mempool_dump(stdout, rte_mp);
1077         }
1078         return rte_mp;
1079 }
1080
1081 /*
1082  * Check given socket id is valid or not with NUMA mode,
1083  * if valid, return 0, else return -1
1084  */
1085 static int
1086 check_socket_id(const unsigned int socket_id)
1087 {
1088         static int warning_once = 0;
1089
1090         if (new_socket_id(socket_id)) {
1091                 if (!warning_once && numa_support)
1092                         printf("Warning: NUMA should be configured manually by"
1093                                " using --port-numa-config and"
1094                                " --ring-numa-config parameters along with"
1095                                " --numa.\n");
1096                 warning_once = 1;
1097                 return -1;
1098         }
1099         return 0;
1100 }
1101
1102 /*
1103  * Get the allowed maximum number of RX queues.
1104  * *pid return the port id which has minimal value of
1105  * max_rx_queues in all ports.
1106  */
1107 queueid_t
1108 get_allowed_max_nb_rxq(portid_t *pid)
1109 {
1110         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1111         bool max_rxq_valid = false;
1112         portid_t pi;
1113         struct rte_eth_dev_info dev_info;
1114
1115         RTE_ETH_FOREACH_DEV(pi) {
1116                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1117                         continue;
1118
1119                 max_rxq_valid = true;
1120                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1121                         allowed_max_rxq = dev_info.max_rx_queues;
1122                         *pid = pi;
1123                 }
1124         }
1125         return max_rxq_valid ? allowed_max_rxq : 0;
1126 }
1127
1128 /*
1129  * Check input rxq is valid or not.
1130  * If input rxq is not greater than any of maximum number
1131  * of RX queues of all ports, it is valid.
1132  * if valid, return 0, else return -1
1133  */
1134 int
1135 check_nb_rxq(queueid_t rxq)
1136 {
1137         queueid_t allowed_max_rxq;
1138         portid_t pid = 0;
1139
1140         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1141         if (rxq > allowed_max_rxq) {
1142                 printf("Fail: input rxq (%u) can't be greater "
1143                        "than max_rx_queues (%u) of port %u\n",
1144                        rxq,
1145                        allowed_max_rxq,
1146                        pid);
1147                 return -1;
1148         }
1149         return 0;
1150 }
1151
1152 /*
1153  * Get the allowed maximum number of TX queues.
1154  * *pid return the port id which has minimal value of
1155  * max_tx_queues in all ports.
1156  */
1157 queueid_t
1158 get_allowed_max_nb_txq(portid_t *pid)
1159 {
1160         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1161         bool max_txq_valid = false;
1162         portid_t pi;
1163         struct rte_eth_dev_info dev_info;
1164
1165         RTE_ETH_FOREACH_DEV(pi) {
1166                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1167                         continue;
1168
1169                 max_txq_valid = true;
1170                 if (dev_info.max_tx_queues < allowed_max_txq) {
1171                         allowed_max_txq = dev_info.max_tx_queues;
1172                         *pid = pi;
1173                 }
1174         }
1175         return max_txq_valid ? allowed_max_txq : 0;
1176 }
1177
1178 /*
1179  * Check input txq is valid or not.
1180  * If input txq is not greater than any of maximum number
1181  * of TX queues of all ports, it is valid.
1182  * if valid, return 0, else return -1
1183  */
1184 int
1185 check_nb_txq(queueid_t txq)
1186 {
1187         queueid_t allowed_max_txq;
1188         portid_t pid = 0;
1189
1190         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1191         if (txq > allowed_max_txq) {
1192                 printf("Fail: input txq (%u) can't be greater "
1193                        "than max_tx_queues (%u) of port %u\n",
1194                        txq,
1195                        allowed_max_txq,
1196                        pid);
1197                 return -1;
1198         }
1199         return 0;
1200 }
1201
1202 /*
1203  * Get the allowed maximum number of RXDs of every rx queue.
1204  * *pid return the port id which has minimal value of
1205  * max_rxd in all queues of all ports.
1206  */
1207 static uint16_t
1208 get_allowed_max_nb_rxd(portid_t *pid)
1209 {
1210         uint16_t allowed_max_rxd = UINT16_MAX;
1211         portid_t pi;
1212         struct rte_eth_dev_info dev_info;
1213
1214         RTE_ETH_FOREACH_DEV(pi) {
1215                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1216                         continue;
1217
1218                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1219                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1220                         *pid = pi;
1221                 }
1222         }
1223         return allowed_max_rxd;
1224 }
1225
1226 /*
1227  * Get the allowed minimal number of RXDs of every rx queue.
1228  * *pid return the port id which has minimal value of
1229  * min_rxd in all queues of all ports.
1230  */
1231 static uint16_t
1232 get_allowed_min_nb_rxd(portid_t *pid)
1233 {
1234         uint16_t allowed_min_rxd = 0;
1235         portid_t pi;
1236         struct rte_eth_dev_info dev_info;
1237
1238         RTE_ETH_FOREACH_DEV(pi) {
1239                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1240                         continue;
1241
1242                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1243                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1244                         *pid = pi;
1245                 }
1246         }
1247
1248         return allowed_min_rxd;
1249 }
1250
1251 /*
1252  * Check input rxd is valid or not.
1253  * If input rxd is not greater than any of maximum number
1254  * of RXDs of every Rx queues and is not less than any of
1255  * minimal number of RXDs of every Rx queues, it is valid.
1256  * if valid, return 0, else return -1
1257  */
1258 int
1259 check_nb_rxd(queueid_t rxd)
1260 {
1261         uint16_t allowed_max_rxd;
1262         uint16_t allowed_min_rxd;
1263         portid_t pid = 0;
1264
1265         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1266         if (rxd > allowed_max_rxd) {
1267                 printf("Fail: input rxd (%u) can't be greater "
1268                        "than max_rxds (%u) of port %u\n",
1269                        rxd,
1270                        allowed_max_rxd,
1271                        pid);
1272                 return -1;
1273         }
1274
1275         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1276         if (rxd < allowed_min_rxd) {
1277                 printf("Fail: input rxd (%u) can't be less "
1278                        "than min_rxds (%u) of port %u\n",
1279                        rxd,
1280                        allowed_min_rxd,
1281                        pid);
1282                 return -1;
1283         }
1284
1285         return 0;
1286 }
1287
1288 /*
1289  * Get the allowed maximum number of TXDs of every rx queues.
1290  * *pid return the port id which has minimal value of
1291  * max_txd in every tx queue.
1292  */
1293 static uint16_t
1294 get_allowed_max_nb_txd(portid_t *pid)
1295 {
1296         uint16_t allowed_max_txd = UINT16_MAX;
1297         portid_t pi;
1298         struct rte_eth_dev_info dev_info;
1299
1300         RTE_ETH_FOREACH_DEV(pi) {
1301                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1302                         continue;
1303
1304                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1305                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1306                         *pid = pi;
1307                 }
1308         }
1309         return allowed_max_txd;
1310 }
1311
1312 /*
1313  * Get the allowed maximum number of TXDs of every tx queues.
1314  * *pid return the port id which has minimal value of
1315  * min_txd in every tx queue.
1316  */
1317 static uint16_t
1318 get_allowed_min_nb_txd(portid_t *pid)
1319 {
1320         uint16_t allowed_min_txd = 0;
1321         portid_t pi;
1322         struct rte_eth_dev_info dev_info;
1323
1324         RTE_ETH_FOREACH_DEV(pi) {
1325                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1326                         continue;
1327
1328                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1329                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1330                         *pid = pi;
1331                 }
1332         }
1333
1334         return allowed_min_txd;
1335 }
1336
1337 /*
1338  * Check input txd is valid or not.
1339  * If input txd is not greater than any of maximum number
1340  * of TXDs of every Rx queues, it is valid.
1341  * if valid, return 0, else return -1
1342  */
1343 int
1344 check_nb_txd(queueid_t txd)
1345 {
1346         uint16_t allowed_max_txd;
1347         uint16_t allowed_min_txd;
1348         portid_t pid = 0;
1349
1350         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1351         if (txd > allowed_max_txd) {
1352                 printf("Fail: input txd (%u) can't be greater "
1353                        "than max_txds (%u) of port %u\n",
1354                        txd,
1355                        allowed_max_txd,
1356                        pid);
1357                 return -1;
1358         }
1359
1360         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1361         if (txd < allowed_min_txd) {
1362                 printf("Fail: input txd (%u) can't be less "
1363                        "than min_txds (%u) of port %u\n",
1364                        txd,
1365                        allowed_min_txd,
1366                        pid);
1367                 return -1;
1368         }
1369         return 0;
1370 }
1371
1372
1373 /*
1374  * Get the allowed maximum number of hairpin queues.
1375  * *pid return the port id which has minimal value of
1376  * max_hairpin_queues in all ports.
1377  */
1378 queueid_t
1379 get_allowed_max_nb_hairpinq(portid_t *pid)
1380 {
1381         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1382         portid_t pi;
1383         struct rte_eth_hairpin_cap cap;
1384
1385         RTE_ETH_FOREACH_DEV(pi) {
1386                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1387                         *pid = pi;
1388                         return 0;
1389                 }
1390                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1391                         allowed_max_hairpinq = cap.max_nb_queues;
1392                         *pid = pi;
1393                 }
1394         }
1395         return allowed_max_hairpinq;
1396 }
1397
1398 /*
1399  * Check input hairpin is valid or not.
1400  * If input hairpin is not greater than any of maximum number
1401  * of hairpin queues of all ports, it is valid.
1402  * if valid, return 0, else return -1
1403  */
1404 int
1405 check_nb_hairpinq(queueid_t hairpinq)
1406 {
1407         queueid_t allowed_max_hairpinq;
1408         portid_t pid = 0;
1409
1410         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1411         if (hairpinq > allowed_max_hairpinq) {
1412                 printf("Fail: input hairpin (%u) can't be greater "
1413                        "than max_hairpin_queues (%u) of port %u\n",
1414                        hairpinq, allowed_max_hairpinq, pid);
1415                 return -1;
1416         }
1417         return 0;
1418 }
1419
1420 static void
1421 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1422 {
1423         struct rte_port *port = &ports[pid];
1424         uint16_t data_size;
1425         int ret;
1426         int i;
1427
1428         port->dev_conf.txmode = tx_mode;
1429         port->dev_conf.rxmode = rx_mode;
1430
1431         ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1432         if (ret != 0)
1433                 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1434
1435         ret = update_jumbo_frame_offload(pid);
1436         if (ret != 0)
1437                 printf("Updating jumbo frame offload failed for port %u\n",
1438                         pid);
1439
1440         if (!(port->dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1441                 port->dev_conf.txmode.offloads &=
1442                         ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1443
1444         /* Apply Rx offloads configuration */
1445         for (i = 0; i < port->dev_info.max_rx_queues; i++)
1446                 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads;
1447         /* Apply Tx offloads configuration */
1448         for (i = 0; i < port->dev_info.max_tx_queues; i++)
1449                 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads;
1450
1451         if (eth_link_speed)
1452                 port->dev_conf.link_speeds = eth_link_speed;
1453
1454         /* set flag to initialize port/queue */
1455         port->need_reconfig = 1;
1456         port->need_reconfig_queues = 1;
1457         port->socket_id = socket_id;
1458         port->tx_metadata = 0;
1459
1460         /*
1461          * Check for maximum number of segments per MTU.
1462          * Accordingly update the mbuf data size.
1463          */
1464         if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1465             port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1466                 data_size = rx_mode.max_rx_pkt_len /
1467                         port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1468
1469                 if ((data_size + RTE_PKTMBUF_HEADROOM) > mbuf_data_size[0]) {
1470                         mbuf_data_size[0] = data_size + RTE_PKTMBUF_HEADROOM;
1471                         TESTPMD_LOG(WARNING,
1472                                     "Configured mbuf size of the first segment %hu\n",
1473                                     mbuf_data_size[0]);
1474                 }
1475         }
1476 }
1477
1478 static void
1479 init_config(void)
1480 {
1481         portid_t pid;
1482         struct rte_mempool *mbp;
1483         unsigned int nb_mbuf_per_pool;
1484         lcoreid_t  lc_id;
1485         struct rte_gro_param gro_param;
1486         uint32_t gso_types;
1487
1488         /* Configuration of logical cores. */
1489         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1490                                 sizeof(struct fwd_lcore *) * nb_lcores,
1491                                 RTE_CACHE_LINE_SIZE);
1492         if (fwd_lcores == NULL) {
1493                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1494                                                         "failed\n", nb_lcores);
1495         }
1496         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1497                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1498                                                sizeof(struct fwd_lcore),
1499                                                RTE_CACHE_LINE_SIZE);
1500                 if (fwd_lcores[lc_id] == NULL) {
1501                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1502                                                                 "failed\n");
1503                 }
1504                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1505         }
1506
1507         RTE_ETH_FOREACH_DEV(pid) {
1508                 uint32_t socket_id;
1509
1510                 if (numa_support) {
1511                         socket_id = port_numa[pid];
1512                         if (port_numa[pid] == NUMA_NO_CONFIG) {
1513                                 socket_id = rte_eth_dev_socket_id(pid);
1514
1515                                 /*
1516                                  * if socket_id is invalid,
1517                                  * set to the first available socket.
1518                                  */
1519                                 if (check_socket_id(socket_id) < 0)
1520                                         socket_id = socket_ids[0];
1521                         }
1522                 } else {
1523                         socket_id = (socket_num == UMA_NO_CONFIG) ?
1524                                     0 : socket_num;
1525                 }
1526                 /* Apply default TxRx configuration for all ports */
1527                 init_config_port_offloads(pid, socket_id);
1528         }
1529         /*
1530          * Create pools of mbuf.
1531          * If NUMA support is disabled, create a single pool of mbuf in
1532          * socket 0 memory by default.
1533          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1534          *
1535          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1536          * nb_txd can be configured at run time.
1537          */
1538         if (param_total_num_mbufs)
1539                 nb_mbuf_per_pool = param_total_num_mbufs;
1540         else {
1541                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1542                         (nb_lcores * mb_mempool_cache) +
1543                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1544                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1545         }
1546
1547         if (numa_support) {
1548                 uint8_t i, j;
1549
1550                 for (i = 0; i < num_sockets; i++)
1551                         for (j = 0; j < mbuf_data_size_n; j++)
1552                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1553                                         mbuf_pool_create(mbuf_data_size[j],
1554                                                           nb_mbuf_per_pool,
1555                                                           socket_ids[i], j);
1556         } else {
1557                 uint8_t i;
1558
1559                 for (i = 0; i < mbuf_data_size_n; i++)
1560                         mempools[i] = mbuf_pool_create
1561                                         (mbuf_data_size[i],
1562                                          nb_mbuf_per_pool,
1563                                          socket_num == UMA_NO_CONFIG ?
1564                                          0 : socket_num, i);
1565         }
1566
1567         init_port_config();
1568
1569         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1570                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1571         /*
1572          * Records which Mbuf pool to use by each logical core, if needed.
1573          */
1574         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1575                 mbp = mbuf_pool_find(
1576                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1577
1578                 if (mbp == NULL)
1579                         mbp = mbuf_pool_find(0, 0);
1580                 fwd_lcores[lc_id]->mbp = mbp;
1581                 /* initialize GSO context */
1582                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1583                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1584                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1585                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1586                         RTE_ETHER_CRC_LEN;
1587                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1588         }
1589
1590         fwd_config_setup();
1591
1592         /* create a gro context for each lcore */
1593         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1594         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1595         gro_param.max_item_per_flow = MAX_PKT_BURST;
1596         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1597                 gro_param.socket_id = rte_lcore_to_socket_id(
1598                                 fwd_lcores_cpuids[lc_id]);
1599                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1600                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1601                         rte_exit(EXIT_FAILURE,
1602                                         "rte_gro_ctx_create() failed\n");
1603                 }
1604         }
1605 }
1606
1607
1608 void
1609 reconfig(portid_t new_port_id, unsigned socket_id)
1610 {
1611         /* Reconfiguration of Ethernet ports. */
1612         init_config_port_offloads(new_port_id, socket_id);
1613         init_port_config();
1614 }
1615
1616
1617 int
1618 init_fwd_streams(void)
1619 {
1620         portid_t pid;
1621         struct rte_port *port;
1622         streamid_t sm_id, nb_fwd_streams_new;
1623         queueid_t q;
1624
1625         /* set socket id according to numa or not */
1626         RTE_ETH_FOREACH_DEV(pid) {
1627                 port = &ports[pid];
1628                 if (nb_rxq > port->dev_info.max_rx_queues) {
1629                         printf("Fail: nb_rxq(%d) is greater than "
1630                                 "max_rx_queues(%d)\n", nb_rxq,
1631                                 port->dev_info.max_rx_queues);
1632                         return -1;
1633                 }
1634                 if (nb_txq > port->dev_info.max_tx_queues) {
1635                         printf("Fail: nb_txq(%d) is greater than "
1636                                 "max_tx_queues(%d)\n", nb_txq,
1637                                 port->dev_info.max_tx_queues);
1638                         return -1;
1639                 }
1640                 if (numa_support) {
1641                         if (port_numa[pid] != NUMA_NO_CONFIG)
1642                                 port->socket_id = port_numa[pid];
1643                         else {
1644                                 port->socket_id = rte_eth_dev_socket_id(pid);
1645
1646                                 /*
1647                                  * if socket_id is invalid,
1648                                  * set to the first available socket.
1649                                  */
1650                                 if (check_socket_id(port->socket_id) < 0)
1651                                         port->socket_id = socket_ids[0];
1652                         }
1653                 }
1654                 else {
1655                         if (socket_num == UMA_NO_CONFIG)
1656                                 port->socket_id = 0;
1657                         else
1658                                 port->socket_id = socket_num;
1659                 }
1660         }
1661
1662         q = RTE_MAX(nb_rxq, nb_txq);
1663         if (q == 0) {
1664                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1665                 return -1;
1666         }
1667         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1668         if (nb_fwd_streams_new == nb_fwd_streams)
1669                 return 0;
1670         /* clear the old */
1671         if (fwd_streams != NULL) {
1672                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1673                         if (fwd_streams[sm_id] == NULL)
1674                                 continue;
1675                         rte_free(fwd_streams[sm_id]);
1676                         fwd_streams[sm_id] = NULL;
1677                 }
1678                 rte_free(fwd_streams);
1679                 fwd_streams = NULL;
1680         }
1681
1682         /* init new */
1683         nb_fwd_streams = nb_fwd_streams_new;
1684         if (nb_fwd_streams) {
1685                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1686                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1687                         RTE_CACHE_LINE_SIZE);
1688                 if (fwd_streams == NULL)
1689                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1690                                  " (struct fwd_stream *)) failed\n",
1691                                  nb_fwd_streams);
1692
1693                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1694                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1695                                 " struct fwd_stream", sizeof(struct fwd_stream),
1696                                 RTE_CACHE_LINE_SIZE);
1697                         if (fwd_streams[sm_id] == NULL)
1698                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1699                                          "(struct fwd_stream) failed\n");
1700                 }
1701         }
1702
1703         return 0;
1704 }
1705
1706 static void
1707 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1708 {
1709         uint64_t total_burst, sburst;
1710         uint64_t nb_burst;
1711         uint64_t burst_stats[4];
1712         uint16_t pktnb_stats[4];
1713         uint16_t nb_pkt;
1714         int burst_percent[4], sburstp;
1715         int i;
1716
1717         /*
1718          * First compute the total number of packet bursts and the
1719          * two highest numbers of bursts of the same number of packets.
1720          */
1721         memset(&burst_stats, 0x0, sizeof(burst_stats));
1722         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1723
1724         /* Show stats for 0 burst size always */
1725         total_burst = pbs->pkt_burst_spread[0];
1726         burst_stats[0] = pbs->pkt_burst_spread[0];
1727         pktnb_stats[0] = 0;
1728
1729         /* Find the next 2 burst sizes with highest occurrences. */
1730         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1731                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1732
1733                 if (nb_burst == 0)
1734                         continue;
1735
1736                 total_burst += nb_burst;
1737
1738                 if (nb_burst > burst_stats[1]) {
1739                         burst_stats[2] = burst_stats[1];
1740                         pktnb_stats[2] = pktnb_stats[1];
1741                         burst_stats[1] = nb_burst;
1742                         pktnb_stats[1] = nb_pkt;
1743                 } else if (nb_burst > burst_stats[2]) {
1744                         burst_stats[2] = nb_burst;
1745                         pktnb_stats[2] = nb_pkt;
1746                 }
1747         }
1748         if (total_burst == 0)
1749                 return;
1750
1751         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1752         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1753                 if (i == 3) {
1754                         printf("%d%% of other]\n", 100 - sburstp);
1755                         return;
1756                 }
1757
1758                 sburst += burst_stats[i];
1759                 if (sburst == total_burst) {
1760                         printf("%d%% of %d pkts]\n",
1761                                 100 - sburstp, (int) pktnb_stats[i]);
1762                         return;
1763                 }
1764
1765                 burst_percent[i] =
1766                         (double)burst_stats[i] / total_burst * 100;
1767                 printf("%d%% of %d pkts + ",
1768                         burst_percent[i], (int) pktnb_stats[i]);
1769                 sburstp += burst_percent[i];
1770         }
1771 }
1772
1773 static void
1774 fwd_stream_stats_display(streamid_t stream_id)
1775 {
1776         struct fwd_stream *fs;
1777         static const char *fwd_top_stats_border = "-------";
1778
1779         fs = fwd_streams[stream_id];
1780         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1781             (fs->fwd_dropped == 0))
1782                 return;
1783         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1784                "TX Port=%2d/Queue=%2d %s\n",
1785                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1786                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1787         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1788                " TX-dropped: %-14"PRIu64,
1789                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1790
1791         /* if checksum mode */
1792         if (cur_fwd_eng == &csum_fwd_engine) {
1793                 printf("  RX- bad IP checksum: %-14"PRIu64
1794                        "  Rx- bad L4 checksum: %-14"PRIu64
1795                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1796                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1797                         fs->rx_bad_outer_l4_csum);
1798                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1799                         fs->rx_bad_outer_ip_csum);
1800         } else {
1801                 printf("\n");
1802         }
1803
1804         if (record_burst_stats) {
1805                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1806                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1807         }
1808 }
1809
1810 void
1811 fwd_stats_display(void)
1812 {
1813         static const char *fwd_stats_border = "----------------------";
1814         static const char *acc_stats_border = "+++++++++++++++";
1815         struct {
1816                 struct fwd_stream *rx_stream;
1817                 struct fwd_stream *tx_stream;
1818                 uint64_t tx_dropped;
1819                 uint64_t rx_bad_ip_csum;
1820                 uint64_t rx_bad_l4_csum;
1821                 uint64_t rx_bad_outer_l4_csum;
1822                 uint64_t rx_bad_outer_ip_csum;
1823         } ports_stats[RTE_MAX_ETHPORTS];
1824         uint64_t total_rx_dropped = 0;
1825         uint64_t total_tx_dropped = 0;
1826         uint64_t total_rx_nombuf = 0;
1827         struct rte_eth_stats stats;
1828         uint64_t fwd_cycles = 0;
1829         uint64_t total_recv = 0;
1830         uint64_t total_xmit = 0;
1831         struct rte_port *port;
1832         streamid_t sm_id;
1833         portid_t pt_id;
1834         int i;
1835
1836         memset(ports_stats, 0, sizeof(ports_stats));
1837
1838         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1839                 struct fwd_stream *fs = fwd_streams[sm_id];
1840
1841                 if (cur_fwd_config.nb_fwd_streams >
1842                     cur_fwd_config.nb_fwd_ports) {
1843                         fwd_stream_stats_display(sm_id);
1844                 } else {
1845                         ports_stats[fs->tx_port].tx_stream = fs;
1846                         ports_stats[fs->rx_port].rx_stream = fs;
1847                 }
1848
1849                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1850
1851                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1852                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1853                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1854                                 fs->rx_bad_outer_l4_csum;
1855                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
1856                                 fs->rx_bad_outer_ip_csum;
1857
1858                 if (record_core_cycles)
1859                         fwd_cycles += fs->core_cycles;
1860         }
1861         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1862                 pt_id = fwd_ports_ids[i];
1863                 port = &ports[pt_id];
1864
1865                 rte_eth_stats_get(pt_id, &stats);
1866                 stats.ipackets -= port->stats.ipackets;
1867                 stats.opackets -= port->stats.opackets;
1868                 stats.ibytes -= port->stats.ibytes;
1869                 stats.obytes -= port->stats.obytes;
1870                 stats.imissed -= port->stats.imissed;
1871                 stats.oerrors -= port->stats.oerrors;
1872                 stats.rx_nombuf -= port->stats.rx_nombuf;
1873
1874                 total_recv += stats.ipackets;
1875                 total_xmit += stats.opackets;
1876                 total_rx_dropped += stats.imissed;
1877                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1878                 total_tx_dropped += stats.oerrors;
1879                 total_rx_nombuf  += stats.rx_nombuf;
1880
1881                 printf("\n  %s Forward statistics for port %-2d %s\n",
1882                        fwd_stats_border, pt_id, fwd_stats_border);
1883
1884                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1885                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1886                        stats.ipackets + stats.imissed);
1887
1888                 if (cur_fwd_eng == &csum_fwd_engine) {
1889                         printf("  Bad-ipcsum: %-14"PRIu64
1890                                " Bad-l4csum: %-14"PRIu64
1891                                "Bad-outer-l4csum: %-14"PRIu64"\n",
1892                                ports_stats[pt_id].rx_bad_ip_csum,
1893                                ports_stats[pt_id].rx_bad_l4_csum,
1894                                ports_stats[pt_id].rx_bad_outer_l4_csum);
1895                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
1896                                ports_stats[pt_id].rx_bad_outer_ip_csum);
1897                 }
1898                 if (stats.ierrors + stats.rx_nombuf > 0) {
1899                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
1900                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1901                 }
1902
1903                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1904                        "TX-total: %-"PRIu64"\n",
1905                        stats.opackets, ports_stats[pt_id].tx_dropped,
1906                        stats.opackets + ports_stats[pt_id].tx_dropped);
1907
1908                 if (record_burst_stats) {
1909                         if (ports_stats[pt_id].rx_stream)
1910                                 pkt_burst_stats_display("RX",
1911                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1912                         if (ports_stats[pt_id].tx_stream)
1913                                 pkt_burst_stats_display("TX",
1914                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1915                 }
1916
1917                 printf("  %s--------------------------------%s\n",
1918                        fwd_stats_border, fwd_stats_border);
1919         }
1920
1921         printf("\n  %s Accumulated forward statistics for all ports"
1922                "%s\n",
1923                acc_stats_border, acc_stats_border);
1924         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1925                "%-"PRIu64"\n"
1926                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1927                "%-"PRIu64"\n",
1928                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1929                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1930         if (total_rx_nombuf > 0)
1931                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1932         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1933                "%s\n",
1934                acc_stats_border, acc_stats_border);
1935         if (record_core_cycles) {
1936 #define CYC_PER_MHZ 1E6
1937                 if (total_recv > 0 || total_xmit > 0) {
1938                         uint64_t total_pkts = 0;
1939                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1940                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1941                                 total_pkts = total_xmit;
1942                         else
1943                                 total_pkts = total_recv;
1944
1945                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1946                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1947                                " MHz Clock\n",
1948                                (double) fwd_cycles / total_pkts,
1949                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1950                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1951                 }
1952         }
1953 }
1954
1955 void
1956 fwd_stats_reset(void)
1957 {
1958         streamid_t sm_id;
1959         portid_t pt_id;
1960         int i;
1961
1962         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1963                 pt_id = fwd_ports_ids[i];
1964                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1965         }
1966         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1967                 struct fwd_stream *fs = fwd_streams[sm_id];
1968
1969                 fs->rx_packets = 0;
1970                 fs->tx_packets = 0;
1971                 fs->fwd_dropped = 0;
1972                 fs->rx_bad_ip_csum = 0;
1973                 fs->rx_bad_l4_csum = 0;
1974                 fs->rx_bad_outer_l4_csum = 0;
1975                 fs->rx_bad_outer_ip_csum = 0;
1976
1977                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1978                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1979                 fs->core_cycles = 0;
1980         }
1981 }
1982
1983 static void
1984 flush_fwd_rx_queues(void)
1985 {
1986         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1987         portid_t  rxp;
1988         portid_t port_id;
1989         queueid_t rxq;
1990         uint16_t  nb_rx;
1991         uint16_t  i;
1992         uint8_t   j;
1993         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1994         uint64_t timer_period;
1995
1996         /* convert to number of cycles */
1997         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1998
1999         for (j = 0; j < 2; j++) {
2000                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2001                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2002                                 port_id = fwd_ports_ids[rxp];
2003                                 /**
2004                                 * testpmd can stuck in the below do while loop
2005                                 * if rte_eth_rx_burst() always returns nonzero
2006                                 * packets. So timer is added to exit this loop
2007                                 * after 1sec timer expiry.
2008                                 */
2009                                 prev_tsc = rte_rdtsc();
2010                                 do {
2011                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2012                                                 pkts_burst, MAX_PKT_BURST);
2013                                         for (i = 0; i < nb_rx; i++)
2014                                                 rte_pktmbuf_free(pkts_burst[i]);
2015
2016                                         cur_tsc = rte_rdtsc();
2017                                         diff_tsc = cur_tsc - prev_tsc;
2018                                         timer_tsc += diff_tsc;
2019                                 } while ((nb_rx > 0) &&
2020                                         (timer_tsc < timer_period));
2021                                 timer_tsc = 0;
2022                         }
2023                 }
2024                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2025         }
2026 }
2027
2028 static void
2029 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2030 {
2031         struct fwd_stream **fsm;
2032         streamid_t nb_fs;
2033         streamid_t sm_id;
2034 #ifdef RTE_LIB_BITRATESTATS
2035         uint64_t tics_per_1sec;
2036         uint64_t tics_datum;
2037         uint64_t tics_current;
2038         uint16_t i, cnt_ports;
2039
2040         cnt_ports = nb_ports;
2041         tics_datum = rte_rdtsc();
2042         tics_per_1sec = rte_get_timer_hz();
2043 #endif
2044         fsm = &fwd_streams[fc->stream_idx];
2045         nb_fs = fc->stream_nb;
2046         do {
2047                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2048                         (*pkt_fwd)(fsm[sm_id]);
2049 #ifdef RTE_LIB_BITRATESTATS
2050                 if (bitrate_enabled != 0 &&
2051                                 bitrate_lcore_id == rte_lcore_id()) {
2052                         tics_current = rte_rdtsc();
2053                         if (tics_current - tics_datum >= tics_per_1sec) {
2054                                 /* Periodic bitrate calculation */
2055                                 for (i = 0; i < cnt_ports; i++)
2056                                         rte_stats_bitrate_calc(bitrate_data,
2057                                                 ports_ids[i]);
2058                                 tics_datum = tics_current;
2059                         }
2060                 }
2061 #endif
2062 #ifdef RTE_LIB_LATENCYSTATS
2063                 if (latencystats_enabled != 0 &&
2064                                 latencystats_lcore_id == rte_lcore_id())
2065                         rte_latencystats_update();
2066 #endif
2067
2068         } while (! fc->stopped);
2069 }
2070
2071 static int
2072 start_pkt_forward_on_core(void *fwd_arg)
2073 {
2074         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2075                              cur_fwd_config.fwd_eng->packet_fwd);
2076         return 0;
2077 }
2078
2079 /*
2080  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2081  * Used to start communication flows in network loopback test configurations.
2082  */
2083 static int
2084 run_one_txonly_burst_on_core(void *fwd_arg)
2085 {
2086         struct fwd_lcore *fwd_lc;
2087         struct fwd_lcore tmp_lcore;
2088
2089         fwd_lc = (struct fwd_lcore *) fwd_arg;
2090         tmp_lcore = *fwd_lc;
2091         tmp_lcore.stopped = 1;
2092         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2093         return 0;
2094 }
2095
2096 /*
2097  * Launch packet forwarding:
2098  *     - Setup per-port forwarding context.
2099  *     - launch logical cores with their forwarding configuration.
2100  */
2101 static void
2102 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2103 {
2104         port_fwd_begin_t port_fwd_begin;
2105         unsigned int i;
2106         unsigned int lc_id;
2107         int diag;
2108
2109         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2110         if (port_fwd_begin != NULL) {
2111                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2112                         (*port_fwd_begin)(fwd_ports_ids[i]);
2113         }
2114         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2115                 lc_id = fwd_lcores_cpuids[i];
2116                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2117                         fwd_lcores[i]->stopped = 0;
2118                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2119                                                      fwd_lcores[i], lc_id);
2120                         if (diag != 0)
2121                                 printf("launch lcore %u failed - diag=%d\n",
2122                                        lc_id, diag);
2123                 }
2124         }
2125 }
2126
2127 /*
2128  * Launch packet forwarding configuration.
2129  */
2130 void
2131 start_packet_forwarding(int with_tx_first)
2132 {
2133         port_fwd_begin_t port_fwd_begin;
2134         port_fwd_end_t  port_fwd_end;
2135         unsigned int i;
2136
2137         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2138                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2139
2140         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2141                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2142
2143         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2144                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2145                 (!nb_rxq || !nb_txq))
2146                 rte_exit(EXIT_FAILURE,
2147                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2148                         cur_fwd_eng->fwd_mode_name);
2149
2150         if (all_ports_started() == 0) {
2151                 printf("Not all ports were started\n");
2152                 return;
2153         }
2154         if (test_done == 0) {
2155                 printf("Packet forwarding already started\n");
2156                 return;
2157         }
2158         test_done = 0;
2159
2160         fwd_config_setup();
2161
2162         if(!no_flush_rx)
2163                 flush_fwd_rx_queues();
2164
2165         pkt_fwd_config_display(&cur_fwd_config);
2166         rxtx_config_display();
2167
2168         fwd_stats_reset();
2169         if (with_tx_first) {
2170                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2171                 if (port_fwd_begin != NULL) {
2172                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2173                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2174                 }
2175                 while (with_tx_first--) {
2176                         launch_packet_forwarding(
2177                                         run_one_txonly_burst_on_core);
2178                         rte_eal_mp_wait_lcore();
2179                 }
2180                 port_fwd_end = tx_only_engine.port_fwd_end;
2181                 if (port_fwd_end != NULL) {
2182                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2183                                 (*port_fwd_end)(fwd_ports_ids[i]);
2184                 }
2185         }
2186         launch_packet_forwarding(start_pkt_forward_on_core);
2187 }
2188
2189 void
2190 stop_packet_forwarding(void)
2191 {
2192         port_fwd_end_t port_fwd_end;
2193         lcoreid_t lc_id;
2194         portid_t pt_id;
2195         int i;
2196
2197         if (test_done) {
2198                 printf("Packet forwarding not started\n");
2199                 return;
2200         }
2201         printf("Telling cores to stop...");
2202         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2203                 fwd_lcores[lc_id]->stopped = 1;
2204         printf("\nWaiting for lcores to finish...\n");
2205         rte_eal_mp_wait_lcore();
2206         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2207         if (port_fwd_end != NULL) {
2208                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2209                         pt_id = fwd_ports_ids[i];
2210                         (*port_fwd_end)(pt_id);
2211                 }
2212         }
2213
2214         fwd_stats_display();
2215
2216         printf("\nDone.\n");
2217         test_done = 1;
2218 }
2219
2220 void
2221 dev_set_link_up(portid_t pid)
2222 {
2223         if (rte_eth_dev_set_link_up(pid) < 0)
2224                 printf("\nSet link up fail.\n");
2225 }
2226
2227 void
2228 dev_set_link_down(portid_t pid)
2229 {
2230         if (rte_eth_dev_set_link_down(pid) < 0)
2231                 printf("\nSet link down fail.\n");
2232 }
2233
2234 static int
2235 all_ports_started(void)
2236 {
2237         portid_t pi;
2238         struct rte_port *port;
2239
2240         RTE_ETH_FOREACH_DEV(pi) {
2241                 port = &ports[pi];
2242                 /* Check if there is a port which is not started */
2243                 if ((port->port_status != RTE_PORT_STARTED) &&
2244                         (port->slave_flag == 0))
2245                         return 0;
2246         }
2247
2248         /* No port is not started */
2249         return 1;
2250 }
2251
2252 int
2253 port_is_stopped(portid_t port_id)
2254 {
2255         struct rte_port *port = &ports[port_id];
2256
2257         if ((port->port_status != RTE_PORT_STOPPED) &&
2258             (port->slave_flag == 0))
2259                 return 0;
2260         return 1;
2261 }
2262
2263 int
2264 all_ports_stopped(void)
2265 {
2266         portid_t pi;
2267
2268         RTE_ETH_FOREACH_DEV(pi) {
2269                 if (!port_is_stopped(pi))
2270                         return 0;
2271         }
2272
2273         return 1;
2274 }
2275
2276 int
2277 port_is_started(portid_t port_id)
2278 {
2279         if (port_id_is_invalid(port_id, ENABLED_WARN))
2280                 return 0;
2281
2282         if (ports[port_id].port_status != RTE_PORT_STARTED)
2283                 return 0;
2284
2285         return 1;
2286 }
2287
2288 /* Configure the Rx and Tx hairpin queues for the selected port. */
2289 static int
2290 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2291 {
2292         queueid_t qi;
2293         struct rte_eth_hairpin_conf hairpin_conf = {
2294                 .peer_count = 1,
2295         };
2296         int i;
2297         int diag;
2298         struct rte_port *port = &ports[pi];
2299         uint16_t peer_rx_port = pi;
2300         uint16_t peer_tx_port = pi;
2301         uint32_t manual = 1;
2302         uint32_t tx_exp = hairpin_mode & 0x10;
2303
2304         if (!(hairpin_mode & 0xf)) {
2305                 peer_rx_port = pi;
2306                 peer_tx_port = pi;
2307                 manual = 0;
2308         } else if (hairpin_mode & 0x1) {
2309                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2310                                                        RTE_ETH_DEV_NO_OWNER);
2311                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2312                         peer_tx_port = rte_eth_find_next_owned_by(0,
2313                                                 RTE_ETH_DEV_NO_OWNER);
2314                 if (p_pi != RTE_MAX_ETHPORTS) {
2315                         peer_rx_port = p_pi;
2316                 } else {
2317                         uint16_t next_pi;
2318
2319                         /* Last port will be the peer RX port of the first. */
2320                         RTE_ETH_FOREACH_DEV(next_pi)
2321                                 peer_rx_port = next_pi;
2322                 }
2323                 manual = 1;
2324         } else if (hairpin_mode & 0x2) {
2325                 if (cnt_pi & 0x1) {
2326                         peer_rx_port = p_pi;
2327                 } else {
2328                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2329                                                 RTE_ETH_DEV_NO_OWNER);
2330                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2331                                 peer_rx_port = pi;
2332                 }
2333                 peer_tx_port = peer_rx_port;
2334                 manual = 1;
2335         }
2336
2337         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2338                 hairpin_conf.peers[0].port = peer_rx_port;
2339                 hairpin_conf.peers[0].queue = i + nb_rxq;
2340                 hairpin_conf.manual_bind = !!manual;
2341                 hairpin_conf.tx_explicit = !!tx_exp;
2342                 diag = rte_eth_tx_hairpin_queue_setup
2343                         (pi, qi, nb_txd, &hairpin_conf);
2344                 i++;
2345                 if (diag == 0)
2346                         continue;
2347
2348                 /* Fail to setup rx queue, return */
2349                 if (rte_atomic16_cmpset(&(port->port_status),
2350                                         RTE_PORT_HANDLING,
2351                                         RTE_PORT_STOPPED) == 0)
2352                         printf("Port %d can not be set back "
2353                                         "to stopped\n", pi);
2354                 printf("Fail to configure port %d hairpin "
2355                                 "queues\n", pi);
2356                 /* try to reconfigure queues next time */
2357                 port->need_reconfig_queues = 1;
2358                 return -1;
2359         }
2360         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2361                 hairpin_conf.peers[0].port = peer_tx_port;
2362                 hairpin_conf.peers[0].queue = i + nb_txq;
2363                 hairpin_conf.manual_bind = !!manual;
2364                 hairpin_conf.tx_explicit = !!tx_exp;
2365                 diag = rte_eth_rx_hairpin_queue_setup
2366                         (pi, qi, nb_rxd, &hairpin_conf);
2367                 i++;
2368                 if (diag == 0)
2369                         continue;
2370
2371                 /* Fail to setup rx queue, return */
2372                 if (rte_atomic16_cmpset(&(port->port_status),
2373                                         RTE_PORT_HANDLING,
2374                                         RTE_PORT_STOPPED) == 0)
2375                         printf("Port %d can not be set back "
2376                                         "to stopped\n", pi);
2377                 printf("Fail to configure port %d hairpin "
2378                                 "queues\n", pi);
2379                 /* try to reconfigure queues next time */
2380                 port->need_reconfig_queues = 1;
2381                 return -1;
2382         }
2383         return 0;
2384 }
2385
2386 /* Configure the Rx with optional split. */
2387 int
2388 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2389                uint16_t nb_rx_desc, unsigned int socket_id,
2390                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2391 {
2392         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2393         unsigned int i, mp_n;
2394         int ret;
2395
2396         if (rx_pkt_nb_segs <= 1 ||
2397             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2398                 rx_conf->rx_seg = NULL;
2399                 rx_conf->rx_nseg = 0;
2400                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2401                                              nb_rx_desc, socket_id,
2402                                              rx_conf, mp);
2403                 return ret;
2404         }
2405         for (i = 0; i < rx_pkt_nb_segs; i++) {
2406                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2407                 struct rte_mempool *mpx;
2408                 /*
2409                  * Use last valid pool for the segments with number
2410                  * exceeding the pool index.
2411                  */
2412                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2413                 mpx = mbuf_pool_find(socket_id, mp_n);
2414                 /* Handle zero as mbuf data buffer size. */
2415                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2416                                    rx_pkt_seg_lengths[i] :
2417                                    mbuf_data_size[mp_n];
2418                 rx_seg->offset = i < rx_pkt_nb_offs ?
2419                                    rx_pkt_seg_offsets[i] : 0;
2420                 rx_seg->mp = mpx ? mpx : mp;
2421         }
2422         rx_conf->rx_nseg = rx_pkt_nb_segs;
2423         rx_conf->rx_seg = rx_useg;
2424         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2425                                     socket_id, rx_conf, NULL);
2426         rx_conf->rx_seg = NULL;
2427         rx_conf->rx_nseg = 0;
2428         return ret;
2429 }
2430
2431 int
2432 start_port(portid_t pid)
2433 {
2434         int diag, need_check_link_status = -1;
2435         portid_t pi;
2436         portid_t p_pi = RTE_MAX_ETHPORTS;
2437         portid_t pl[RTE_MAX_ETHPORTS];
2438         portid_t peer_pl[RTE_MAX_ETHPORTS];
2439         uint16_t cnt_pi = 0;
2440         uint16_t cfg_pi = 0;
2441         int peer_pi;
2442         queueid_t qi;
2443         struct rte_port *port;
2444         struct rte_eth_hairpin_cap cap;
2445
2446         if (port_id_is_invalid(pid, ENABLED_WARN))
2447                 return 0;
2448
2449         RTE_ETH_FOREACH_DEV(pi) {
2450                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2451                         continue;
2452
2453                 need_check_link_status = 0;
2454                 port = &ports[pi];
2455                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2456                                                  RTE_PORT_HANDLING) == 0) {
2457                         printf("Port %d is now not stopped\n", pi);
2458                         continue;
2459                 }
2460
2461                 if (port->need_reconfig > 0) {
2462                         port->need_reconfig = 0;
2463
2464                         if (flow_isolate_all) {
2465                                 int ret = port_flow_isolate(pi, 1);
2466                                 if (ret) {
2467                                         printf("Failed to apply isolated"
2468                                                " mode on port %d\n", pi);
2469                                         return -1;
2470                                 }
2471                         }
2472                         configure_rxtx_dump_callbacks(0);
2473                         printf("Configuring Port %d (socket %u)\n", pi,
2474                                         port->socket_id);
2475                         if (nb_hairpinq > 0 &&
2476                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2477                                 printf("Port %d doesn't support hairpin "
2478                                        "queues\n", pi);
2479                                 return -1;
2480                         }
2481                         /* configure port */
2482                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2483                                                      nb_txq + nb_hairpinq,
2484                                                      &(port->dev_conf));
2485                         if (diag != 0) {
2486                                 if (rte_atomic16_cmpset(&(port->port_status),
2487                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2488                                         printf("Port %d can not be set back "
2489                                                         "to stopped\n", pi);
2490                                 printf("Fail to configure port %d\n", pi);
2491                                 /* try to reconfigure port next time */
2492                                 port->need_reconfig = 1;
2493                                 return -1;
2494                         }
2495                 }
2496                 if (port->need_reconfig_queues > 0) {
2497                         port->need_reconfig_queues = 0;
2498                         /* setup tx queues */
2499                         for (qi = 0; qi < nb_txq; qi++) {
2500                                 if ((numa_support) &&
2501                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2502                                         diag = rte_eth_tx_queue_setup(pi, qi,
2503                                                 port->nb_tx_desc[qi],
2504                                                 txring_numa[pi],
2505                                                 &(port->tx_conf[qi]));
2506                                 else
2507                                         diag = rte_eth_tx_queue_setup(pi, qi,
2508                                                 port->nb_tx_desc[qi],
2509                                                 port->socket_id,
2510                                                 &(port->tx_conf[qi]));
2511
2512                                 if (diag == 0)
2513                                         continue;
2514
2515                                 /* Fail to setup tx queue, return */
2516                                 if (rte_atomic16_cmpset(&(port->port_status),
2517                                                         RTE_PORT_HANDLING,
2518                                                         RTE_PORT_STOPPED) == 0)
2519                                         printf("Port %d can not be set back "
2520                                                         "to stopped\n", pi);
2521                                 printf("Fail to configure port %d tx queues\n",
2522                                        pi);
2523                                 /* try to reconfigure queues next time */
2524                                 port->need_reconfig_queues = 1;
2525                                 return -1;
2526                         }
2527                         for (qi = 0; qi < nb_rxq; qi++) {
2528                                 /* setup rx queues */
2529                                 if ((numa_support) &&
2530                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2531                                         struct rte_mempool * mp =
2532                                                 mbuf_pool_find
2533                                                         (rxring_numa[pi], 0);
2534                                         if (mp == NULL) {
2535                                                 printf("Failed to setup RX queue:"
2536                                                         "No mempool allocation"
2537                                                         " on the socket %d\n",
2538                                                         rxring_numa[pi]);
2539                                                 return -1;
2540                                         }
2541
2542                                         diag = rx_queue_setup(pi, qi,
2543                                              port->nb_rx_desc[qi],
2544                                              rxring_numa[pi],
2545                                              &(port->rx_conf[qi]),
2546                                              mp);
2547                                 } else {
2548                                         struct rte_mempool *mp =
2549                                                 mbuf_pool_find
2550                                                         (port->socket_id, 0);
2551                                         if (mp == NULL) {
2552                                                 printf("Failed to setup RX queue:"
2553                                                         "No mempool allocation"
2554                                                         " on the socket %d\n",
2555                                                         port->socket_id);
2556                                                 return -1;
2557                                         }
2558                                         diag = rx_queue_setup(pi, qi,
2559                                              port->nb_rx_desc[qi],
2560                                              port->socket_id,
2561                                              &(port->rx_conf[qi]),
2562                                              mp);
2563                                 }
2564                                 if (diag == 0)
2565                                         continue;
2566
2567                                 /* Fail to setup rx queue, return */
2568                                 if (rte_atomic16_cmpset(&(port->port_status),
2569                                                         RTE_PORT_HANDLING,
2570                                                         RTE_PORT_STOPPED) == 0)
2571                                         printf("Port %d can not be set back "
2572                                                         "to stopped\n", pi);
2573                                 printf("Fail to configure port %d rx queues\n",
2574                                        pi);
2575                                 /* try to reconfigure queues next time */
2576                                 port->need_reconfig_queues = 1;
2577                                 return -1;
2578                         }
2579                         /* setup hairpin queues */
2580                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2581                                 return -1;
2582                 }
2583                 configure_rxtx_dump_callbacks(verbose_level);
2584                 if (clear_ptypes) {
2585                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2586                                         NULL, 0);
2587                         if (diag < 0)
2588                                 printf(
2589                                 "Port %d: Failed to disable Ptype parsing\n",
2590                                 pi);
2591                 }
2592
2593                 p_pi = pi;
2594                 cnt_pi++;
2595
2596                 /* start port */
2597                 diag = rte_eth_dev_start(pi);
2598                 if (diag < 0) {
2599                         printf("Fail to start port %d: %s\n", pi,
2600                                rte_strerror(-diag));
2601
2602                         /* Fail to setup rx queue, return */
2603                         if (rte_atomic16_cmpset(&(port->port_status),
2604                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2605                                 printf("Port %d can not be set back to "
2606                                                         "stopped\n", pi);
2607                         continue;
2608                 }
2609
2610                 if (rte_atomic16_cmpset(&(port->port_status),
2611                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2612                         printf("Port %d can not be set into started\n", pi);
2613
2614                 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
2615                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2616                                 port->eth_addr.addr_bytes[0],
2617                                 port->eth_addr.addr_bytes[1],
2618                                 port->eth_addr.addr_bytes[2],
2619                                 port->eth_addr.addr_bytes[3],
2620                                 port->eth_addr.addr_bytes[4],
2621                                 port->eth_addr.addr_bytes[5]);
2622
2623                 /* at least one port started, need checking link status */
2624                 need_check_link_status = 1;
2625
2626                 pl[cfg_pi++] = pi;
2627         }
2628
2629         if (need_check_link_status == 1 && !no_link_check)
2630                 check_all_ports_link_status(RTE_PORT_ALL);
2631         else if (need_check_link_status == 0)
2632                 printf("Please stop the ports first\n");
2633
2634         if (hairpin_mode & 0xf) {
2635                 uint16_t i;
2636                 int j;
2637
2638                 /* bind all started hairpin ports */
2639                 for (i = 0; i < cfg_pi; i++) {
2640                         pi = pl[i];
2641                         /* bind current Tx to all peer Rx */
2642                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2643                                                         RTE_MAX_ETHPORTS, 1);
2644                         if (peer_pi < 0)
2645                                 return peer_pi;
2646                         for (j = 0; j < peer_pi; j++) {
2647                                 if (!port_is_started(peer_pl[j]))
2648                                         continue;
2649                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2650                                 if (diag < 0) {
2651                                         printf("Error during binding hairpin"
2652                                                " Tx port %u to %u: %s\n",
2653                                                pi, peer_pl[j],
2654                                                rte_strerror(-diag));
2655                                         return -1;
2656                                 }
2657                         }
2658                         /* bind all peer Tx to current Rx */
2659                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2660                                                         RTE_MAX_ETHPORTS, 0);
2661                         if (peer_pi < 0)
2662                                 return peer_pi;
2663                         for (j = 0; j < peer_pi; j++) {
2664                                 if (!port_is_started(peer_pl[j]))
2665                                         continue;
2666                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2667                                 if (diag < 0) {
2668                                         printf("Error during binding hairpin"
2669                                                " Tx port %u to %u: %s\n",
2670                                                peer_pl[j], pi,
2671                                                rte_strerror(-diag));
2672                                         return -1;
2673                                 }
2674                         }
2675                 }
2676         }
2677
2678         printf("Done\n");
2679         return 0;
2680 }
2681
2682 void
2683 stop_port(portid_t pid)
2684 {
2685         portid_t pi;
2686         struct rte_port *port;
2687         int need_check_link_status = 0;
2688         portid_t peer_pl[RTE_MAX_ETHPORTS];
2689         int peer_pi;
2690
2691         if (port_id_is_invalid(pid, ENABLED_WARN))
2692                 return;
2693
2694         printf("Stopping ports...\n");
2695
2696         RTE_ETH_FOREACH_DEV(pi) {
2697                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2698                         continue;
2699
2700                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2701                         printf("Please remove port %d from forwarding configuration.\n", pi);
2702                         continue;
2703                 }
2704
2705                 if (port_is_bonding_slave(pi)) {
2706                         printf("Please remove port %d from bonded device.\n", pi);
2707                         continue;
2708                 }
2709
2710                 port = &ports[pi];
2711                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2712                                                 RTE_PORT_HANDLING) == 0)
2713                         continue;
2714
2715                 if (hairpin_mode & 0xf) {
2716                         int j;
2717
2718                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2719                         /* unbind all peer Tx from current Rx */
2720                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2721                                                         RTE_MAX_ETHPORTS, 0);
2722                         if (peer_pi < 0)
2723                                 continue;
2724                         for (j = 0; j < peer_pi; j++) {
2725                                 if (!port_is_started(peer_pl[j]))
2726                                         continue;
2727                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2728                         }
2729                 }
2730
2731                 if (port->flow_list)
2732                         port_flow_flush(pi);
2733
2734                 if (rte_eth_dev_stop(pi) != 0)
2735                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2736                                 pi);
2737
2738                 if (rte_atomic16_cmpset(&(port->port_status),
2739                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2740                         printf("Port %d can not be set into stopped\n", pi);
2741                 need_check_link_status = 1;
2742         }
2743         if (need_check_link_status && !no_link_check)
2744                 check_all_ports_link_status(RTE_PORT_ALL);
2745
2746         printf("Done\n");
2747 }
2748
2749 static void
2750 remove_invalid_ports_in(portid_t *array, portid_t *total)
2751 {
2752         portid_t i;
2753         portid_t new_total = 0;
2754
2755         for (i = 0; i < *total; i++)
2756                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2757                         array[new_total] = array[i];
2758                         new_total++;
2759                 }
2760         *total = new_total;
2761 }
2762
2763 static void
2764 remove_invalid_ports(void)
2765 {
2766         remove_invalid_ports_in(ports_ids, &nb_ports);
2767         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2768         nb_cfg_ports = nb_fwd_ports;
2769 }
2770
2771 void
2772 close_port(portid_t pid)
2773 {
2774         portid_t pi;
2775         struct rte_port *port;
2776
2777         if (port_id_is_invalid(pid, ENABLED_WARN))
2778                 return;
2779
2780         printf("Closing ports...\n");
2781
2782         RTE_ETH_FOREACH_DEV(pi) {
2783                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2784                         continue;
2785
2786                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2787                         printf("Please remove port %d from forwarding configuration.\n", pi);
2788                         continue;
2789                 }
2790
2791                 if (port_is_bonding_slave(pi)) {
2792                         printf("Please remove port %d from bonded device.\n", pi);
2793                         continue;
2794                 }
2795
2796                 port = &ports[pi];
2797                 if (rte_atomic16_cmpset(&(port->port_status),
2798                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2799                         printf("Port %d is already closed\n", pi);
2800                         continue;
2801                 }
2802
2803                 port_flow_flush(pi);
2804                 rte_eth_dev_close(pi);
2805         }
2806
2807         remove_invalid_ports();
2808         printf("Done\n");
2809 }
2810
2811 void
2812 reset_port(portid_t pid)
2813 {
2814         int diag;
2815         portid_t pi;
2816         struct rte_port *port;
2817
2818         if (port_id_is_invalid(pid, ENABLED_WARN))
2819                 return;
2820
2821         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2822                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2823                 printf("Can not reset port(s), please stop port(s) first.\n");
2824                 return;
2825         }
2826
2827         printf("Resetting ports...\n");
2828
2829         RTE_ETH_FOREACH_DEV(pi) {
2830                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2831                         continue;
2832
2833                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2834                         printf("Please remove port %d from forwarding "
2835                                "configuration.\n", pi);
2836                         continue;
2837                 }
2838
2839                 if (port_is_bonding_slave(pi)) {
2840                         printf("Please remove port %d from bonded device.\n",
2841                                pi);
2842                         continue;
2843                 }
2844
2845                 diag = rte_eth_dev_reset(pi);
2846                 if (diag == 0) {
2847                         port = &ports[pi];
2848                         port->need_reconfig = 1;
2849                         port->need_reconfig_queues = 1;
2850                 } else {
2851                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2852                 }
2853         }
2854
2855         printf("Done\n");
2856 }
2857
2858 void
2859 attach_port(char *identifier)
2860 {
2861         portid_t pi;
2862         struct rte_dev_iterator iterator;
2863
2864         printf("Attaching a new port...\n");
2865
2866         if (identifier == NULL) {
2867                 printf("Invalid parameters are specified\n");
2868                 return;
2869         }
2870
2871         if (rte_dev_probe(identifier) < 0) {
2872                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2873                 return;
2874         }
2875
2876         /* first attach mode: event */
2877         if (setup_on_probe_event) {
2878                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2879                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2880                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2881                                         ports[pi].need_setup != 0)
2882                                 setup_attached_port(pi);
2883                 return;
2884         }
2885
2886         /* second attach mode: iterator */
2887         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2888                 /* setup ports matching the devargs used for probing */
2889                 if (port_is_forwarding(pi))
2890                         continue; /* port was already attached before */
2891                 setup_attached_port(pi);
2892         }
2893 }
2894
2895 static void
2896 setup_attached_port(portid_t pi)
2897 {
2898         unsigned int socket_id;
2899         int ret;
2900
2901         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2902         /* if socket_id is invalid, set to the first available socket. */
2903         if (check_socket_id(socket_id) < 0)
2904                 socket_id = socket_ids[0];
2905         reconfig(pi, socket_id);
2906         ret = rte_eth_promiscuous_enable(pi);
2907         if (ret != 0)
2908                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2909                         pi, rte_strerror(-ret));
2910
2911         ports_ids[nb_ports++] = pi;
2912         fwd_ports_ids[nb_fwd_ports++] = pi;
2913         nb_cfg_ports = nb_fwd_ports;
2914         ports[pi].need_setup = 0;
2915         ports[pi].port_status = RTE_PORT_STOPPED;
2916
2917         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2918         printf("Done\n");
2919 }
2920
2921 static void
2922 detach_device(struct rte_device *dev)
2923 {
2924         portid_t sibling;
2925
2926         if (dev == NULL) {
2927                 printf("Device already removed\n");
2928                 return;
2929         }
2930
2931         printf("Removing a device...\n");
2932
2933         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2934                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2935                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2936                                 printf("Port %u not stopped\n", sibling);
2937                                 return;
2938                         }
2939                         port_flow_flush(sibling);
2940                 }
2941         }
2942
2943         if (rte_dev_remove(dev) < 0) {
2944                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2945                 return;
2946         }
2947         remove_invalid_ports();
2948
2949         printf("Device is detached\n");
2950         printf("Now total ports is %d\n", nb_ports);
2951         printf("Done\n");
2952         return;
2953 }
2954
2955 void
2956 detach_port_device(portid_t port_id)
2957 {
2958         if (port_id_is_invalid(port_id, ENABLED_WARN))
2959                 return;
2960
2961         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2962                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2963                         printf("Port not stopped\n");
2964                         return;
2965                 }
2966                 printf("Port was not closed\n");
2967         }
2968
2969         detach_device(rte_eth_devices[port_id].device);
2970 }
2971
2972 void
2973 detach_devargs(char *identifier)
2974 {
2975         struct rte_dev_iterator iterator;
2976         struct rte_devargs da;
2977         portid_t port_id;
2978
2979         printf("Removing a device...\n");
2980
2981         memset(&da, 0, sizeof(da));
2982         if (rte_devargs_parsef(&da, "%s", identifier)) {
2983                 printf("cannot parse identifier\n");
2984                 return;
2985         }
2986
2987         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2988                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2989                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2990                                 printf("Port %u not stopped\n", port_id);
2991                                 rte_eth_iterator_cleanup(&iterator);
2992                                 rte_devargs_reset(&da);
2993                                 return;
2994                         }
2995                         port_flow_flush(port_id);
2996                 }
2997         }
2998
2999         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3000                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3001                             da.name, da.bus->name);
3002                 rte_devargs_reset(&da);
3003                 return;
3004         }
3005
3006         remove_invalid_ports();
3007
3008         printf("Device %s is detached\n", identifier);
3009         printf("Now total ports is %d\n", nb_ports);
3010         printf("Done\n");
3011         rte_devargs_reset(&da);
3012 }
3013
3014 void
3015 pmd_test_exit(void)
3016 {
3017         portid_t pt_id;
3018         unsigned int i;
3019         int ret;
3020
3021         if (test_done == 0)
3022                 stop_packet_forwarding();
3023
3024 #ifndef RTE_EXEC_ENV_WINDOWS
3025         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3026                 if (mempools[i]) {
3027                         if (mp_alloc_type == MP_ALLOC_ANON)
3028                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3029                                                      NULL);
3030                 }
3031         }
3032 #endif
3033         if (ports != NULL) {
3034                 no_link_check = 1;
3035                 RTE_ETH_FOREACH_DEV(pt_id) {
3036                         printf("\nStopping port %d...\n", pt_id);
3037                         fflush(stdout);
3038                         stop_port(pt_id);
3039                 }
3040                 RTE_ETH_FOREACH_DEV(pt_id) {
3041                         printf("\nShutting down port %d...\n", pt_id);
3042                         fflush(stdout);
3043                         close_port(pt_id);
3044                 }
3045         }
3046
3047         if (hot_plug) {
3048                 ret = rte_dev_event_monitor_stop();
3049                 if (ret) {
3050                         RTE_LOG(ERR, EAL,
3051                                 "fail to stop device event monitor.");
3052                         return;
3053                 }
3054
3055                 ret = rte_dev_event_callback_unregister(NULL,
3056                         dev_event_callback, NULL);
3057                 if (ret < 0) {
3058                         RTE_LOG(ERR, EAL,
3059                                 "fail to unregister device event callback.\n");
3060                         return;
3061                 }
3062
3063                 ret = rte_dev_hotplug_handle_disable();
3064                 if (ret) {
3065                         RTE_LOG(ERR, EAL,
3066                                 "fail to disable hotplug handling.\n");
3067                         return;
3068                 }
3069         }
3070         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3071                 if (mempools[i])
3072                         rte_mempool_free(mempools[i]);
3073         }
3074
3075         printf("\nBye...\n");
3076 }
3077
3078 typedef void (*cmd_func_t)(void);
3079 struct pmd_test_command {
3080         const char *cmd_name;
3081         cmd_func_t cmd_func;
3082 };
3083
3084 /* Check the link status of all ports in up to 9s, and print them finally */
3085 static void
3086 check_all_ports_link_status(uint32_t port_mask)
3087 {
3088 #define CHECK_INTERVAL 100 /* 100ms */
3089 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3090         portid_t portid;
3091         uint8_t count, all_ports_up, print_flag = 0;
3092         struct rte_eth_link link;
3093         int ret;
3094         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3095
3096         printf("Checking link statuses...\n");
3097         fflush(stdout);
3098         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3099                 all_ports_up = 1;
3100                 RTE_ETH_FOREACH_DEV(portid) {
3101                         if ((port_mask & (1 << portid)) == 0)
3102                                 continue;
3103                         memset(&link, 0, sizeof(link));
3104                         ret = rte_eth_link_get_nowait(portid, &link);
3105                         if (ret < 0) {
3106                                 all_ports_up = 0;
3107                                 if (print_flag == 1)
3108                                         printf("Port %u link get failed: %s\n",
3109                                                 portid, rte_strerror(-ret));
3110                                 continue;
3111                         }
3112                         /* print link status if flag set */
3113                         if (print_flag == 1) {
3114                                 rte_eth_link_to_str(link_status,
3115                                         sizeof(link_status), &link);
3116                                 printf("Port %d %s\n", portid, link_status);
3117                                 continue;
3118                         }
3119                         /* clear all_ports_up flag if any link down */
3120                         if (link.link_status == ETH_LINK_DOWN) {
3121                                 all_ports_up = 0;
3122                                 break;
3123                         }
3124                 }
3125                 /* after finally printing all link status, get out */
3126                 if (print_flag == 1)
3127                         break;
3128
3129                 if (all_ports_up == 0) {
3130                         fflush(stdout);
3131                         rte_delay_ms(CHECK_INTERVAL);
3132                 }
3133
3134                 /* set the print_flag if all ports up or timeout */
3135                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3136                         print_flag = 1;
3137                 }
3138
3139                 if (lsc_interrupt)
3140                         break;
3141         }
3142 }
3143
3144 static void
3145 rmv_port_callback(void *arg)
3146 {
3147         int need_to_start = 0;
3148         int org_no_link_check = no_link_check;
3149         portid_t port_id = (intptr_t)arg;
3150         struct rte_device *dev;
3151
3152         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3153
3154         if (!test_done && port_is_forwarding(port_id)) {
3155                 need_to_start = 1;
3156                 stop_packet_forwarding();
3157         }
3158         no_link_check = 1;
3159         stop_port(port_id);
3160         no_link_check = org_no_link_check;
3161
3162         /* Save rte_device pointer before closing ethdev port */
3163         dev = rte_eth_devices[port_id].device;
3164         close_port(port_id);
3165         detach_device(dev); /* might be already removed or have more ports */
3166
3167         if (need_to_start)
3168                 start_packet_forwarding(0);
3169 }
3170
3171 /* This function is used by the interrupt thread */
3172 static int
3173 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3174                   void *ret_param)
3175 {
3176         RTE_SET_USED(param);
3177         RTE_SET_USED(ret_param);
3178
3179         if (type >= RTE_ETH_EVENT_MAX) {
3180                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3181                         port_id, __func__, type);
3182                 fflush(stderr);
3183         } else if (event_print_mask & (UINT32_C(1) << type)) {
3184                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3185                         eth_event_desc[type]);
3186                 fflush(stdout);
3187         }
3188
3189         switch (type) {
3190         case RTE_ETH_EVENT_NEW:
3191                 ports[port_id].need_setup = 1;
3192                 ports[port_id].port_status = RTE_PORT_HANDLING;
3193                 break;
3194         case RTE_ETH_EVENT_INTR_RMV:
3195                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3196                         break;
3197                 if (rte_eal_alarm_set(100000,
3198                                 rmv_port_callback, (void *)(intptr_t)port_id))
3199                         fprintf(stderr, "Could not set up deferred device removal\n");
3200                 break;
3201         case RTE_ETH_EVENT_DESTROY:
3202                 ports[port_id].port_status = RTE_PORT_CLOSED;
3203                 printf("Port %u is closed\n", port_id);
3204                 break;
3205         default:
3206                 break;
3207         }
3208         return 0;
3209 }
3210
3211 static int
3212 register_eth_event_callback(void)
3213 {
3214         int ret;
3215         enum rte_eth_event_type event;
3216
3217         for (event = RTE_ETH_EVENT_UNKNOWN;
3218                         event < RTE_ETH_EVENT_MAX; event++) {
3219                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3220                                 event,
3221                                 eth_event_callback,
3222                                 NULL);
3223                 if (ret != 0) {
3224                         TESTPMD_LOG(ERR, "Failed to register callback for "
3225                                         "%s event\n", eth_event_desc[event]);
3226                         return -1;
3227                 }
3228         }
3229
3230         return 0;
3231 }
3232
3233 /* This function is used by the interrupt thread */
3234 static void
3235 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3236                              __rte_unused void *arg)
3237 {
3238         uint16_t port_id;
3239         int ret;
3240
3241         if (type >= RTE_DEV_EVENT_MAX) {
3242                 fprintf(stderr, "%s called upon invalid event %d\n",
3243                         __func__, type);
3244                 fflush(stderr);
3245         }
3246
3247         switch (type) {
3248         case RTE_DEV_EVENT_REMOVE:
3249                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3250                         device_name);
3251                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3252                 if (ret) {
3253                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3254                                 device_name);
3255                         return;
3256                 }
3257                 /*
3258                  * Because the user's callback is invoked in eal interrupt
3259                  * callback, the interrupt callback need to be finished before
3260                  * it can be unregistered when detaching device. So finish
3261                  * callback soon and use a deferred removal to detach device
3262                  * is need. It is a workaround, once the device detaching be
3263                  * moved into the eal in the future, the deferred removal could
3264                  * be deleted.
3265                  */
3266                 if (rte_eal_alarm_set(100000,
3267                                 rmv_port_callback, (void *)(intptr_t)port_id))
3268                         RTE_LOG(ERR, EAL,
3269                                 "Could not set up deferred device removal\n");
3270                 break;
3271         case RTE_DEV_EVENT_ADD:
3272                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3273                         device_name);
3274                 /* TODO: After finish kernel driver binding,
3275                  * begin to attach port.
3276                  */
3277                 break;
3278         default:
3279                 break;
3280         }
3281 }
3282
3283 static void
3284 rxtx_port_config(struct rte_port *port)
3285 {
3286         uint16_t qid;
3287         uint64_t offloads;
3288
3289         for (qid = 0; qid < nb_rxq; qid++) {
3290                 offloads = port->rx_conf[qid].offloads;
3291                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3292                 if (offloads != 0)
3293                         port->rx_conf[qid].offloads = offloads;
3294
3295                 /* Check if any Rx parameters have been passed */
3296                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3297                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3298
3299                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3300                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3301
3302                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3303                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3304
3305                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3306                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3307
3308                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3309                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3310
3311                 port->nb_rx_desc[qid] = nb_rxd;
3312         }
3313
3314         for (qid = 0; qid < nb_txq; qid++) {
3315                 offloads = port->tx_conf[qid].offloads;
3316                 port->tx_conf[qid] = port->dev_info.default_txconf;
3317                 if (offloads != 0)
3318                         port->tx_conf[qid].offloads = offloads;
3319
3320                 /* Check if any Tx parameters have been passed */
3321                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3322                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3323
3324                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3325                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3326
3327                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3328                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3329
3330                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3331                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3332
3333                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3334                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3335
3336                 port->nb_tx_desc[qid] = nb_txd;
3337         }
3338 }
3339
3340 /*
3341  * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3342  * MTU is also aligned if JUMBO_FRAME offload is not set.
3343  *
3344  * port->dev_info should be set before calling this function.
3345  *
3346  * return 0 on success, negative on error
3347  */
3348 int
3349 update_jumbo_frame_offload(portid_t portid)
3350 {
3351         struct rte_port *port = &ports[portid];
3352         uint32_t eth_overhead;
3353         uint64_t rx_offloads;
3354         int ret;
3355         bool on;
3356
3357         /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3358         if (port->dev_info.max_mtu != UINT16_MAX &&
3359             port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3360                 eth_overhead = port->dev_info.max_rx_pktlen -
3361                                 port->dev_info.max_mtu;
3362         else
3363                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3364
3365         rx_offloads = port->dev_conf.rxmode.offloads;
3366
3367         /* Default config value is 0 to use PMD specific overhead */
3368         if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3369                 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3370
3371         if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3372                 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3373                 on = false;
3374         } else {
3375                 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3376                         printf("Frame size (%u) is not supported by port %u\n",
3377                                 port->dev_conf.rxmode.max_rx_pkt_len,
3378                                 portid);
3379                         return -1;
3380                 }
3381                 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3382                 on = true;
3383         }
3384
3385         if (rx_offloads != port->dev_conf.rxmode.offloads) {
3386                 uint16_t qid;
3387
3388                 port->dev_conf.rxmode.offloads = rx_offloads;
3389
3390                 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3391                 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3392                         if (on)
3393                                 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3394                         else
3395                                 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3396                 }
3397         }
3398
3399         /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3400          * if unset do it here
3401          */
3402         if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3403                 ret = rte_eth_dev_set_mtu(portid,
3404                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3405                 if (ret)
3406                         printf("Failed to set MTU to %u for port %u\n",
3407                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3408                                 portid);
3409         }
3410
3411         return 0;
3412 }
3413
3414 void
3415 init_port_config(void)
3416 {
3417         portid_t pid;
3418         struct rte_port *port;
3419         int ret;
3420
3421         RTE_ETH_FOREACH_DEV(pid) {
3422                 port = &ports[pid];
3423                 port->dev_conf.fdir_conf = fdir_conf;
3424
3425                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3426                 if (ret != 0)
3427                         return;
3428
3429                 if (nb_rxq > 1) {
3430                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3431                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3432                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3433                 } else {
3434                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3435                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3436                 }
3437
3438                 if (port->dcb_flag == 0) {
3439                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3440                                 port->dev_conf.rxmode.mq_mode =
3441                                         (enum rte_eth_rx_mq_mode)
3442                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3443                         else
3444                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3445                 }
3446
3447                 rxtx_port_config(port);
3448
3449                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3450                 if (ret != 0)
3451                         return;
3452
3453 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3454                 rte_pmd_ixgbe_bypass_init(pid);
3455 #endif
3456
3457                 if (lsc_interrupt &&
3458                     (rte_eth_devices[pid].data->dev_flags &
3459                      RTE_ETH_DEV_INTR_LSC))
3460                         port->dev_conf.intr_conf.lsc = 1;
3461                 if (rmv_interrupt &&
3462                     (rte_eth_devices[pid].data->dev_flags &
3463                      RTE_ETH_DEV_INTR_RMV))
3464                         port->dev_conf.intr_conf.rmv = 1;
3465         }
3466 }
3467
3468 void set_port_slave_flag(portid_t slave_pid)
3469 {
3470         struct rte_port *port;
3471
3472         port = &ports[slave_pid];
3473         port->slave_flag = 1;
3474 }
3475
3476 void clear_port_slave_flag(portid_t slave_pid)
3477 {
3478         struct rte_port *port;
3479
3480         port = &ports[slave_pid];
3481         port->slave_flag = 0;
3482 }
3483
3484 uint8_t port_is_bonding_slave(portid_t slave_pid)
3485 {
3486         struct rte_port *port;
3487
3488         port = &ports[slave_pid];
3489         if ((rte_eth_devices[slave_pid].data->dev_flags &
3490             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3491                 return 1;
3492         return 0;
3493 }
3494
3495 const uint16_t vlan_tags[] = {
3496                 0,  1,  2,  3,  4,  5,  6,  7,
3497                 8,  9, 10, 11,  12, 13, 14, 15,
3498                 16, 17, 18, 19, 20, 21, 22, 23,
3499                 24, 25, 26, 27, 28, 29, 30, 31
3500 };
3501
3502 static  int
3503 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3504                  enum dcb_mode_enable dcb_mode,
3505                  enum rte_eth_nb_tcs num_tcs,
3506                  uint8_t pfc_en)
3507 {
3508         uint8_t i;
3509         int32_t rc;
3510         struct rte_eth_rss_conf rss_conf;
3511
3512         /*
3513          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3514          * given above, and the number of traffic classes available for use.
3515          */
3516         if (dcb_mode == DCB_VT_ENABLED) {
3517                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3518                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3519                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3520                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3521
3522                 /* VMDQ+DCB RX and TX configurations */
3523                 vmdq_rx_conf->enable_default_pool = 0;
3524                 vmdq_rx_conf->default_pool = 0;
3525                 vmdq_rx_conf->nb_queue_pools =
3526                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3527                 vmdq_tx_conf->nb_queue_pools =
3528                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3529
3530                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3531                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3532                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3533                         vmdq_rx_conf->pool_map[i].pools =
3534                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3535                 }
3536                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3537                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3538                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3539                 }
3540
3541                 /* set DCB mode of RX and TX of multiple queues */
3542                 eth_conf->rxmode.mq_mode =
3543                                 (enum rte_eth_rx_mq_mode)
3544                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3545                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3546         } else {
3547                 struct rte_eth_dcb_rx_conf *rx_conf =
3548                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3549                 struct rte_eth_dcb_tx_conf *tx_conf =
3550                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3551
3552                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3553
3554                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3555                 if (rc != 0)
3556                         return rc;
3557
3558                 rx_conf->nb_tcs = num_tcs;
3559                 tx_conf->nb_tcs = num_tcs;
3560
3561                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3562                         rx_conf->dcb_tc[i] = i % num_tcs;
3563                         tx_conf->dcb_tc[i] = i % num_tcs;
3564                 }
3565
3566                 eth_conf->rxmode.mq_mode =
3567                                 (enum rte_eth_rx_mq_mode)
3568                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3569                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3570                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3571         }
3572
3573         if (pfc_en)
3574                 eth_conf->dcb_capability_en =
3575                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3576         else
3577                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3578
3579         return 0;
3580 }
3581
3582 int
3583 init_port_dcb_config(portid_t pid,
3584                      enum dcb_mode_enable dcb_mode,
3585                      enum rte_eth_nb_tcs num_tcs,
3586                      uint8_t pfc_en)
3587 {
3588         struct rte_eth_conf port_conf;
3589         struct rte_port *rte_port;
3590         int retval;
3591         uint16_t i;
3592
3593         rte_port = &ports[pid];
3594
3595         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3596
3597         port_conf.rxmode = rte_port->dev_conf.rxmode;
3598         port_conf.txmode = rte_port->dev_conf.txmode;
3599
3600         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3601         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3602         if (retval < 0)
3603                 return retval;
3604         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3605
3606         /* re-configure the device . */
3607         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3608         if (retval < 0)
3609                 return retval;
3610
3611         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3612         if (retval != 0)
3613                 return retval;
3614
3615         /* If dev_info.vmdq_pool_base is greater than 0,
3616          * the queue id of vmdq pools is started after pf queues.
3617          */
3618         if (dcb_mode == DCB_VT_ENABLED &&
3619             rte_port->dev_info.vmdq_pool_base > 0) {
3620                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3621                         " for port %d.", pid);
3622                 return -1;
3623         }
3624
3625         /* Assume the ports in testpmd have the same dcb capability
3626          * and has the same number of rxq and txq in dcb mode
3627          */
3628         if (dcb_mode == DCB_VT_ENABLED) {
3629                 if (rte_port->dev_info.max_vfs > 0) {
3630                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3631                         nb_txq = rte_port->dev_info.nb_tx_queues;
3632                 } else {
3633                         nb_rxq = rte_port->dev_info.max_rx_queues;
3634                         nb_txq = rte_port->dev_info.max_tx_queues;
3635                 }
3636         } else {
3637                 /*if vt is disabled, use all pf queues */
3638                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3639                         nb_rxq = rte_port->dev_info.max_rx_queues;
3640                         nb_txq = rte_port->dev_info.max_tx_queues;
3641                 } else {
3642                         nb_rxq = (queueid_t)num_tcs;
3643                         nb_txq = (queueid_t)num_tcs;
3644
3645                 }
3646         }
3647         rx_free_thresh = 64;
3648
3649         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3650
3651         rxtx_port_config(rte_port);
3652         /* VLAN filter */
3653         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3654         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3655                 rx_vft_set(pid, vlan_tags[i], 1);
3656
3657         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3658         if (retval != 0)
3659                 return retval;
3660
3661         rte_port->dcb_flag = 1;
3662
3663         /* Enter DCB configuration status */
3664         dcb_config = 1;
3665
3666         return 0;
3667 }
3668
3669 static void
3670 init_port(void)
3671 {
3672         int i;
3673
3674         /* Configuration of Ethernet ports. */
3675         ports = rte_zmalloc("testpmd: ports",
3676                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3677                             RTE_CACHE_LINE_SIZE);
3678         if (ports == NULL) {
3679                 rte_exit(EXIT_FAILURE,
3680                                 "rte_zmalloc(%d struct rte_port) failed\n",
3681                                 RTE_MAX_ETHPORTS);
3682         }
3683         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3684                 LIST_INIT(&ports[i].flow_tunnel_list);
3685         /* Initialize ports NUMA structures */
3686         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3687         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3688         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3689 }
3690
3691 static void
3692 force_quit(void)
3693 {
3694         pmd_test_exit();
3695         prompt_exit();
3696 }
3697
3698 static void
3699 print_stats(void)
3700 {
3701         uint8_t i;
3702         const char clr[] = { 27, '[', '2', 'J', '\0' };
3703         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3704
3705         /* Clear screen and move to top left */
3706         printf("%s%s", clr, top_left);
3707
3708         printf("\nPort statistics ====================================");
3709         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3710                 nic_stats_display(fwd_ports_ids[i]);
3711
3712         fflush(stdout);
3713 }
3714
3715 static void
3716 signal_handler(int signum)
3717 {
3718         if (signum == SIGINT || signum == SIGTERM) {
3719                 printf("\nSignal %d received, preparing to exit...\n",
3720                                 signum);
3721 #ifdef RTE_LIB_PDUMP
3722                 /* uninitialize packet capture framework */
3723                 rte_pdump_uninit();
3724 #endif
3725 #ifdef RTE_LIB_LATENCYSTATS
3726                 if (latencystats_enabled != 0)
3727                         rte_latencystats_uninit();
3728 #endif
3729                 force_quit();
3730                 /* Set flag to indicate the force termination. */
3731                 f_quit = 1;
3732                 /* exit with the expected status */
3733 #ifndef RTE_EXEC_ENV_WINDOWS
3734                 signal(signum, SIG_DFL);
3735                 kill(getpid(), signum);
3736 #endif
3737         }
3738 }
3739
3740 int
3741 main(int argc, char** argv)
3742 {
3743         int diag;
3744         portid_t port_id;
3745         uint16_t count;
3746         int ret;
3747
3748         signal(SIGINT, signal_handler);
3749         signal(SIGTERM, signal_handler);
3750
3751         testpmd_logtype = rte_log_register("testpmd");
3752         if (testpmd_logtype < 0)
3753                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3754         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3755
3756         diag = rte_eal_init(argc, argv);
3757         if (diag < 0)
3758                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3759                          rte_strerror(rte_errno));
3760
3761         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3762                 rte_exit(EXIT_FAILURE,
3763                          "Secondary process type not supported.\n");
3764
3765         ret = register_eth_event_callback();
3766         if (ret != 0)
3767                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3768
3769 #ifdef RTE_LIB_PDUMP
3770         /* initialize packet capture framework */
3771         rte_pdump_init();
3772 #endif
3773
3774         count = 0;
3775         RTE_ETH_FOREACH_DEV(port_id) {
3776                 ports_ids[count] = port_id;
3777                 count++;
3778         }
3779         nb_ports = (portid_t) count;
3780         if (nb_ports == 0)
3781                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3782
3783         /* allocate port structures, and init them */
3784         init_port();
3785
3786         set_def_fwd_config();
3787         if (nb_lcores == 0)
3788                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3789                          "Check the core mask argument\n");
3790
3791         /* Bitrate/latency stats disabled by default */
3792 #ifdef RTE_LIB_BITRATESTATS
3793         bitrate_enabled = 0;
3794 #endif
3795 #ifdef RTE_LIB_LATENCYSTATS
3796         latencystats_enabled = 0;
3797 #endif
3798
3799         /* on FreeBSD, mlockall() is disabled by default */
3800 #ifdef RTE_EXEC_ENV_FREEBSD
3801         do_mlockall = 0;
3802 #else
3803         do_mlockall = 1;
3804 #endif
3805
3806         argc -= diag;
3807         argv += diag;
3808         if (argc > 1)
3809                 launch_args_parse(argc, argv);
3810
3811 #ifndef RTE_EXEC_ENV_WINDOWS
3812         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3813                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3814                         strerror(errno));
3815         }
3816 #endif
3817
3818         if (tx_first && interactive)
3819                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3820                                 "interactive mode.\n");
3821
3822         if (tx_first && lsc_interrupt) {
3823                 printf("Warning: lsc_interrupt needs to be off when "
3824                                 " using tx_first. Disabling.\n");
3825                 lsc_interrupt = 0;
3826         }
3827
3828         if (!nb_rxq && !nb_txq)
3829                 printf("Warning: Either rx or tx queues should be non-zero\n");
3830
3831         if (nb_rxq > 1 && nb_rxq > nb_txq)
3832                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3833                        "but nb_txq=%d will prevent to fully test it.\n",
3834                        nb_rxq, nb_txq);
3835
3836         init_config();
3837
3838         if (hot_plug) {
3839                 ret = rte_dev_hotplug_handle_enable();
3840                 if (ret) {
3841                         RTE_LOG(ERR, EAL,
3842                                 "fail to enable hotplug handling.");
3843                         return -1;
3844                 }
3845
3846                 ret = rte_dev_event_monitor_start();
3847                 if (ret) {
3848                         RTE_LOG(ERR, EAL,
3849                                 "fail to start device event monitoring.");
3850                         return -1;
3851                 }
3852
3853                 ret = rte_dev_event_callback_register(NULL,
3854                         dev_event_callback, NULL);
3855                 if (ret) {
3856                         RTE_LOG(ERR, EAL,
3857                                 "fail  to register device event callback\n");
3858                         return -1;
3859                 }
3860         }
3861
3862         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3863                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3864
3865         /* set all ports to promiscuous mode by default */
3866         RTE_ETH_FOREACH_DEV(port_id) {
3867                 ret = rte_eth_promiscuous_enable(port_id);
3868                 if (ret != 0)
3869                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3870                                 port_id, rte_strerror(-ret));
3871         }
3872
3873         /* Init metrics library */
3874         rte_metrics_init(rte_socket_id());
3875
3876 #ifdef RTE_LIB_LATENCYSTATS
3877         if (latencystats_enabled != 0) {
3878                 int ret = rte_latencystats_init(1, NULL);
3879                 if (ret)
3880                         printf("Warning: latencystats init()"
3881                                 " returned error %d\n", ret);
3882                 printf("Latencystats running on lcore %d\n",
3883                         latencystats_lcore_id);
3884         }
3885 #endif
3886
3887         /* Setup bitrate stats */
3888 #ifdef RTE_LIB_BITRATESTATS
3889         if (bitrate_enabled != 0) {
3890                 bitrate_data = rte_stats_bitrate_create();
3891                 if (bitrate_data == NULL)
3892                         rte_exit(EXIT_FAILURE,
3893                                 "Could not allocate bitrate data.\n");
3894                 rte_stats_bitrate_reg(bitrate_data);
3895         }
3896 #endif
3897
3898 #ifdef RTE_LIB_CMDLINE
3899         if (strlen(cmdline_filename) != 0)
3900                 cmdline_read_from_file(cmdline_filename);
3901
3902         if (interactive == 1) {
3903                 if (auto_start) {
3904                         printf("Start automatic packet forwarding\n");
3905                         start_packet_forwarding(0);
3906                 }
3907                 prompt();
3908                 pmd_test_exit();
3909         } else
3910 #endif
3911         {
3912                 char c;
3913                 int rc;
3914
3915                 f_quit = 0;
3916
3917                 printf("No commandline core given, start packet forwarding\n");
3918                 start_packet_forwarding(tx_first);
3919                 if (stats_period != 0) {
3920                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3921                         uint64_t timer_period;
3922
3923                         /* Convert to number of cycles */
3924                         timer_period = stats_period * rte_get_timer_hz();
3925
3926                         while (f_quit == 0) {
3927                                 cur_time = rte_get_timer_cycles();
3928                                 diff_time += cur_time - prev_time;
3929
3930                                 if (diff_time >= timer_period) {
3931                                         print_stats();
3932                                         /* Reset the timer */
3933                                         diff_time = 0;
3934                                 }
3935                                 /* Sleep to avoid unnecessary checks */
3936                                 prev_time = cur_time;
3937                                 rte_delay_us_sleep(US_PER_S);
3938                         }
3939                 }
3940
3941                 printf("Press enter to exit\n");
3942                 rc = read(0, &c, 1);
3943                 pmd_test_exit();
3944                 if (rc < 0)
3945                         return 1;
3946         }
3947
3948         ret = rte_eal_cleanup();
3949         if (ret != 0)
3950                 rte_exit(EXIT_FAILURE,
3951                          "EAL cleanup failed: %s\n", strerror(-ret));
3952
3953         return EXIT_SUCCESS;
3954 }