app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #ifndef RTE_EXEC_ENV_WINDOWS
  13 #include <sys/mman.h>
  14 #endif
  15 #include <sys/types.h>
  16 #include <errno.h>
  17 #include <stdbool.h>
  18
  19 #include <sys/queue.h>
  20 #include <sys/stat.h>
  21
  22 #include <stdint.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_byteorder.h>
  29 #include <rte_log.h>
  30 #include <rte_debug.h>
  31 #include <rte_cycles.h>
  32 #include <rte_memory.h>
  33 #include <rte_memcpy.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_alarm.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_atomic.h>
  40 #include <rte_branch_prediction.h>
  41 #include <rte_mempool.h>
  42 #include <rte_malloc.h>
  43 #include <rte_mbuf.h>
  44 #include <rte_mbuf_pool_ops.h>
  45 #include <rte_interrupts.h>
  46 #include <rte_pci.h>
  47 #include <rte_ether.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_dev.h>
  50 #include <rte_string_fns.h>
  51 #ifdef RTE_NET_IXGBE
  52 #include <rte_pmd_ixgbe.h>
  53 #endif
  54 #ifdef RTE_LIB_PDUMP
  55 #include <rte_pdump.h>
  56 #endif
  57 #include <rte_flow.h>
  58 #include <rte_metrics.h>
  59 #ifdef RTE_LIB_BITRATESTATS
  60 #include <rte_bitrate.h>
  61 #endif
  62 #ifdef RTE_LIB_LATENCYSTATS
  63 #include <rte_latencystats.h>
  64 #endif
  65 #ifdef RTE_EXEC_ENV_WINDOWS
  66 #include <process.h>
  67 #endif
  68
  69 #include "testpmd.h"
  70
  71 #ifndef MAP_HUGETLB
  72 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  73 #define HUGE_FLAG (0x40000)
  74 #else
  75 #define HUGE_FLAG MAP_HUGETLB
  76 #endif
  77
  78 #ifndef MAP_HUGE_SHIFT
  79 /* older kernels (or FreeBSD) will not have this define */
  80 #define HUGE_SHIFT (26)
  81 #else
  82 #define HUGE_SHIFT MAP_HUGE_SHIFT
  83 #endif
  84
  85 #define EXTMEM_HEAP_NAME "extmem"
  86 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  87
  88 uint16_t verbose_level = 0; /**< Silent by default. */
  89 int testpmd_logtype; /**< Log type for testpmd logs */
  90
  91 /* use main core for command line ? */
  92 uint8_t interactive = 0;
  93 uint8_t auto_start = 0;
  94 uint8_t tx_first;
  95 char cmdline_filename[PATH_MAX] = {0};
  96
  97 /*
  98  * NUMA support configuration.
  99  * When set, the NUMA support attempts to dispatch the allocation of the
 100  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 101  * probed ports among the CPU sockets 0 and 1.
 102  * Otherwise, all memory is allocated from CPU socket 0.
 103  */
 104 uint8_t numa_support = 1; /**< numa enabled by default */
 105
 106 /*
 107  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 108  * not configured.
 109  */
 110 uint8_t socket_num = UMA_NO_CONFIG;
 111
 112 /*
 113  * Select mempool allocation type:
 114  * - native: use regular DPDK memory
 115  * - anon: use regular DPDK memory to create mempool, but populate using
 116  *         anonymous memory (may not be IOVA-contiguous)
 117  * - xmem: use externally allocated hugepage memory
 118  */
 119 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 120
 121 /*
 122  * Store specified sockets on which memory pool to be used by ports
 123  * is allocated.
 124  */
 125 uint8_t port_numa[RTE_MAX_ETHPORTS];
 126
 127 /*
 128  * Store specified sockets on which RX ring to be used by ports
 129  * is allocated.
 130  */
 131 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 132
 133 /*
 134  * Store specified sockets on which TX ring to be used by ports
 135  * is allocated.
 136  */
 137 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 138
 139 /*
 140  * Record the Ethernet address of peer target ports to which packets are
 141  * forwarded.
 142  * Must be instantiated with the ethernet addresses of peer traffic generator
 143  * ports.
 144  */
 145 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 146 portid_t nb_peer_eth_addrs = 0;
 147
 148 /*
 149  * Probed Target Environment.
 150  */
 151 struct rte_port *ports;        /**< For all probed ethernet ports. */
 152 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 153 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 154 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 155
 156 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 157
 158 /*
 159  * Test Forwarding Configuration.
 160  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 161  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 162  */
 163 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 164 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 165 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 166 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 167
 168 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 169 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 170
 171 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 172 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 173
 174 /*
 175  * Forwarding engines.
 176  */
 177 struct fwd_engine * fwd_engines[] = {
 178         &io_fwd_engine,
 179         &mac_fwd_engine,
 180         &mac_swap_engine,
 181         &flow_gen_engine,
 182         &rx_only_engine,
 183         &tx_only_engine,
 184         &csum_fwd_engine,
 185         &icmp_echo_engine,
 186         &noisy_vnf_engine,
 187         &five_tuple_swap_fwd_engine,
 188 #ifdef RTE_LIBRTE_IEEE1588
 189         &ieee1588_fwd_engine,
 190 #endif
 191         NULL,
 192 };
 193
 194 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 195 uint16_t mempool_flags;
 196
 197 struct fwd_config cur_fwd_config;
 198 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 199 uint32_t retry_enabled;
 200 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 201 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 202
 203 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 204 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 205         DEFAULT_MBUF_DATA_SIZE
 206 }; /**< Mbuf data space size. */
 207 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 208                                       * specified on command-line. */
 209 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 210
 211 /*
 212  * In container, it cannot terminate the process which running with 'stats-period'
 213  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 214  */
 215 uint8_t f_quit;
 216
 217 /*
 218  * Configuration of packet segments used to scatter received packets
 219  * if some of split features is configured.
 220  */
 221 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 222 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 223 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 224 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 225
 226 /*
 227  * Configuration of packet segments used by the "txonly" processing engine.
 228  */
 229 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 230 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 231         TXONLY_DEF_PACKET_LEN,
 232 };
 233 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 234
 235 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 236 /**< Split policy for packets to TX. */
 237
 238 uint8_t txonly_multi_flow;
 239 /**< Whether multiple flows are generated in TXONLY mode. */
 240
 241 uint32_t tx_pkt_times_inter;
 242 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 243
 244 uint32_t tx_pkt_times_intra;
 245 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 246
 247 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 248 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 249 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 250
 251 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 252 uint8_t dcb_config = 0;
 253
 254 /*
 255  * Configurable number of RX/TX queues.
 256  */
 257 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 258 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 259 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 260
 261 /*
 262  * Configurable number of RX/TX ring descriptors.
 263  * Defaults are supplied by drivers via ethdev.
 264  */
 265 #define RTE_TEST_RX_DESC_DEFAULT 0
 266 #define RTE_TEST_TX_DESC_DEFAULT 0
 267 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 268 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 269
 270 #define RTE_PMD_PARAM_UNSET -1
 271 /*
 272  * Configurable values of RX and TX ring threshold registers.
 273  */
 274
 275 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 276 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 277 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 278
 279 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 280 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 281 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of RX free threshold.
 285  */
 286 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 287
 288 /*
 289  * Configurable value of RX drop enable.
 290  */
 291 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 292
 293 /*
 294  * Configurable value of TX free threshold.
 295  */
 296 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 297
 298 /*
 299  * Configurable value of TX RS bit threshold.
 300  */
 301 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 302
 303 /*
 304  * Configurable value of buffered packets before sending.
 305  */
 306 uint16_t noisy_tx_sw_bufsz;
 307
 308 /*
 309  * Configurable value of packet buffer timeout.
 310  */
 311 uint16_t noisy_tx_sw_buf_flush_time;
 312
 313 /*
 314  * Configurable value for size of VNF internal memory area
 315  * used for simulating noisy neighbour behaviour
 316  */
 317 uint64_t noisy_lkup_mem_sz;
 318
 319 /*
 320  * Configurable value of number of random writes done in
 321  * VNF simulation memory area.
 322  */
 323 uint64_t noisy_lkup_num_writes;
 324
 325 /*
 326  * Configurable value of number of random reads done in
 327  * VNF simulation memory area.
 328  */
 329 uint64_t noisy_lkup_num_reads;
 330
 331 /*
 332  * Configurable value of number of random reads/writes done in
 333  * VNF simulation memory area.
 334  */
 335 uint64_t noisy_lkup_num_reads_writes;
 336
 337 /*
 338  * Receive Side Scaling (RSS) configuration.
 339  */
 340 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 341
 342 /*
 343  * Port topology configuration
 344  */
 345 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 346
 347 /*
 348  * Avoids to flush all the RX streams before starts forwarding.
 349  */
 350 uint8_t no_flush_rx = 0; /* flush by default */
 351
 352 /*
 353  * Flow API isolated mode.
 354  */
 355 uint8_t flow_isolate_all;
 356
 357 /*
 358  * Avoids to check link status when starting/stopping a port.
 359  */
 360 uint8_t no_link_check = 0; /* check by default */
 361
 362 /*
 363  * Don't automatically start all ports in interactive mode.
 364  */
 365 uint8_t no_device_start = 0;
 366
 367 /*
 368  * Enable link status change notification
 369  */
 370 uint8_t lsc_interrupt = 1; /* enabled by default */
 371
 372 /*
 373  * Enable device removal notification.
 374  */
 375 uint8_t rmv_interrupt = 1; /* enabled by default */
 376
 377 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 378
 379 /* After attach, port setup is called on event or by iterator */
 380 bool setup_on_probe_event = true;
 381
 382 /* Clear ptypes on port initialization. */
 383 uint8_t clear_ptypes = true;
 384
 385 /* Hairpin ports configuration mode. */
 386 uint16_t hairpin_mode;
 387
 388 /* Pretty printing of ethdev events */
 389 static const char * const eth_event_desc[] = {
 390         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 391         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 392         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 393         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 394         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 395         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 396         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 397         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 398         [RTE_ETH_EVENT_NEW] = "device probed",
 399         [RTE_ETH_EVENT_DESTROY] = "device released",
 400         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 401         [RTE_ETH_EVENT_MAX] = NULL,
 402 };
 403
 404 /*
 405  * Display or mask ether events
 406  * Default to all events except VF_MBOX
 407  */
 408 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 413                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 414                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 415                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 416 /*
 417  * Decide if all memory are locked for performance.
 418  */
 419 int do_mlockall = 0;
 420
 421 /*
 422  * NIC bypass mode configuration options.
 423  */
 424
 425 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 426 /* The NIC bypass watchdog timeout. */
 427 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 428 #endif
 429
 430
 431 #ifdef RTE_LIB_LATENCYSTATS
 432
 433 /*
 434  * Set when latency stats is enabled in the commandline
 435  */
 436 uint8_t latencystats_enabled;
 437
 438 /*
 439  * Lcore ID to serive latency statistics.
 440  */
 441 lcoreid_t latencystats_lcore_id = -1;
 442
 443 #endif
 444
 445 /*
 446  * Ethernet device configuration.
 447  */
 448 struct rte_eth_rxmode rx_mode = {
 449         /* Default maximum frame length.
 450          * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
 451          * in init_config().
 452          */
 453         .max_rx_pkt_len = 0,
 454 };
 455
 456 struct rte_eth_txmode tx_mode = {
 457         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 458 };
 459
 460 struct rte_fdir_conf fdir_conf = {
 461         .mode = RTE_FDIR_MODE_NONE,
 462         .pballoc = RTE_FDIR_PBALLOC_64K,
 463         .status = RTE_FDIR_REPORT_STATUS,
 464         .mask = {
 465                 .vlan_tci_mask = 0xFFEF,
 466                 .ipv4_mask     = {
 467                         .src_ip = 0xFFFFFFFF,
 468                         .dst_ip = 0xFFFFFFFF,
 469                 },
 470                 .ipv6_mask     = {
 471                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 472                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 473                 },
 474                 .src_port_mask = 0xFFFF,
 475                 .dst_port_mask = 0xFFFF,
 476                 .mac_addr_byte_mask = 0xFF,
 477                 .tunnel_type_mask = 1,
 478                 .tunnel_id_mask = 0xFFFFFFFF,
 479         },
 480         .drop_queue = 127,
 481 };
 482
 483 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 484
 485 /*
 486  * Display zero values by default for xstats
 487  */
 488 uint8_t xstats_hide_zero;
 489
 490 /*
 491  * Measure of CPU cycles disabled by default
 492  */
 493 uint8_t record_core_cycles;
 494
 495 /*
 496  * Display of RX and TX bursts disabled by default
 497  */
 498 uint8_t record_burst_stats;
 499
 500 unsigned int num_sockets = 0;
 501 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 502
 503 #ifdef RTE_LIB_BITRATESTATS
 504 /* Bitrate statistics */
 505 struct rte_stats_bitrates *bitrate_data;
 506 lcoreid_t bitrate_lcore_id;
 507 uint8_t bitrate_enabled;
 508 #endif
 509
 510 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 511 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 512
 513 /*
 514  * hexadecimal bitmask of RX mq mode can be enabled.
 515  */
 516 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 517
 518 /*
 519  * Used to set forced link speed
 520  */
 521 uint32_t eth_link_speed;
 522
 523 /* Forward function declarations */
 524 static void setup_attached_port(portid_t pi);
 525 static void check_all_ports_link_status(uint32_t port_mask);
 526 static int eth_event_callback(portid_t port_id,
 527                               enum rte_eth_event_type type,
 528                               void *param, void *ret_param);
 529 static void dev_event_callback(const char *device_name,
 530                                 enum rte_dev_event_type type,
 531                                 void *param);
 532
 533 /*
 534  * Check if all the ports are started.
 535  * If yes, return positive value. If not, return zero.
 536  */
 537 static int all_ports_started(void);
 538
 539 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 540 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 541
 542 /* Holds the registered mbuf dynamic flags names. */
 543 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 544
 545 /*
 546  * Helper function to check if socket is already discovered.
 547  * If yes, return positive value. If not, return zero.
 548  */
 549 int
 550 new_socket_id(unsigned int socket_id)
 551 {
 552         unsigned int i;
 553
 554         for (i = 0; i < num_sockets; i++) {
 555                 if (socket_ids[i] == socket_id)
 556                         return 0;
 557         }
 558         return 1;
 559 }
 560
 561 /*
 562  * Setup default configuration.
 563  */
 564 static void
 565 set_default_fwd_lcores_config(void)
 566 {
 567         unsigned int i;
 568         unsigned int nb_lc;
 569         unsigned int sock_num;
 570
 571         nb_lc = 0;
 572         for (i = 0; i < RTE_MAX_LCORE; i++) {
 573                 if (!rte_lcore_is_enabled(i))
 574                         continue;
 575                 sock_num = rte_lcore_to_socket_id(i);
 576                 if (new_socket_id(sock_num)) {
 577                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 578                                 rte_exit(EXIT_FAILURE,
 579                                          "Total sockets greater than %u\n",
 580                                          RTE_MAX_NUMA_NODES);
 581                         }
 582                         socket_ids[num_sockets++] = sock_num;
 583                 }
 584                 if (i == rte_get_main_lcore())
 585                         continue;
 586                 fwd_lcores_cpuids[nb_lc++] = i;
 587         }
 588         nb_lcores = (lcoreid_t) nb_lc;
 589         nb_cfg_lcores = nb_lcores;
 590         nb_fwd_lcores = 1;
 591 }
 592
 593 static void
 594 set_def_peer_eth_addrs(void)
 595 {
 596         portid_t i;
 597
 598         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 599                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 600                 peer_eth_addrs[i].addr_bytes[5] = i;
 601         }
 602 }
 603
 604 static void
 605 set_default_fwd_ports_config(void)
 606 {
 607         portid_t pt_id;
 608         int i = 0;
 609
 610         RTE_ETH_FOREACH_DEV(pt_id) {
 611                 fwd_ports_ids[i++] = pt_id;
 612
 613                 /* Update sockets info according to the attached device */
 614                 int socket_id = rte_eth_dev_socket_id(pt_id);
 615                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 616                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 617                                 rte_exit(EXIT_FAILURE,
 618                                          "Total sockets greater than %u\n",
 619                                          RTE_MAX_NUMA_NODES);
 620                         }
 621                         socket_ids[num_sockets++] = socket_id;
 622                 }
 623         }
 624
 625         nb_cfg_ports = nb_ports;
 626         nb_fwd_ports = nb_ports;
 627 }
 628
 629 void
 630 set_def_fwd_config(void)
 631 {
 632         set_default_fwd_lcores_config();
 633         set_def_peer_eth_addrs();
 634         set_default_fwd_ports_config();
 635 }
 636
 637 #ifndef RTE_EXEC_ENV_WINDOWS
 638 /* extremely pessimistic estimation of memory required to create a mempool */
 639 static int
 640 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 641 {
 642         unsigned int n_pages, mbuf_per_pg, leftover;
 643         uint64_t total_mem, mbuf_mem, obj_sz;
 644
 645         /* there is no good way to predict how much space the mempool will
 646          * occupy because it will allocate chunks on the fly, and some of those
 647          * will come from default DPDK memory while some will come from our
 648          * external memory, so just assume 128MB will be enough for everyone.
 649          */
 650         uint64_t hdr_mem = 128 << 20;
 651
 652         /* account for possible non-contiguousness */
 653         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 654         if (obj_sz > pgsz) {
 655                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 656                 return -1;
 657         }
 658
 659         mbuf_per_pg = pgsz / obj_sz;
 660         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 661         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 662
 663         mbuf_mem = n_pages * pgsz;
 664
 665         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 666
 667         if (total_mem > SIZE_MAX) {
 668                 TESTPMD_LOG(ERR, "Memory size too big\n");
 669                 return -1;
 670         }
 671         *out = (size_t)total_mem;
 672
 673         return 0;
 674 }
 675
 676 static int
 677 pagesz_flags(uint64_t page_sz)
 678 {
 679         /* as per mmap() manpage, all page sizes are log2 of page size
 680          * shifted by MAP_HUGE_SHIFT
 681          */
 682         int log2 = rte_log2_u64(page_sz);
 683
 684         return (log2 << HUGE_SHIFT);
 685 }
 686
 687 static void *
 688 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 689 {
 690         void *addr;
 691         int flags;
 692
 693         /* allocate anonymous hugepages */
 694         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 695         if (huge)
 696                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 697
 698         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 699         if (addr == MAP_FAILED)
 700                 return NULL;
 701
 702         return addr;
 703 }
 704
 705 struct extmem_param {
 706         void *addr;
 707         size_t len;
 708         size_t pgsz;
 709         rte_iova_t *iova_table;
 710         unsigned int iova_table_len;
 711 };
 712
 713 static int
 714 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 715                 bool huge)
 716 {
 717         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 718                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 719         unsigned int cur_page, n_pages, pgsz_idx;
 720         size_t mem_sz, cur_pgsz;
 721         rte_iova_t *iovas = NULL;
 722         void *addr;
 723         int ret;
 724
 725         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 726                 /* skip anything that is too big */
 727                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 728                         continue;
 729
 730                 cur_pgsz = pgsizes[pgsz_idx];
 731
 732                 /* if we were told not to allocate hugepages, override */
 733                 if (!huge)
 734                         cur_pgsz = sysconf(_SC_PAGESIZE);
 735
 736                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 737                 if (ret < 0) {
 738                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 739                         return -1;
 740                 }
 741
 742                 /* allocate our memory */
 743                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 744
 745                 /* if we couldn't allocate memory with a specified page size,
 746                  * that doesn't mean we can't do it with other page sizes, so
 747                  * try another one.
 748                  */
 749                 if (addr == NULL)
 750                         continue;
 751
 752                 /* store IOVA addresses for every page in this memory area */
 753                 n_pages = mem_sz / cur_pgsz;
 754
 755                 iovas = malloc(sizeof(*iovas) * n_pages);
 756
 757                 if (iovas == NULL) {
 758                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 759                         goto fail;
 760                 }
 761                 /* lock memory if it's not huge pages */
 762                 if (!huge)
 763                         mlock(addr, mem_sz);
 764
 765                 /* populate IOVA addresses */
 766                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 767                         rte_iova_t iova;
 768                         size_t offset;
 769                         void *cur;
 770
 771                         offset = cur_pgsz * cur_page;
 772                         cur = RTE_PTR_ADD(addr, offset);
 773
 774                         /* touch the page before getting its IOVA */
 775                         *(volatile char *)cur = 0;
 776
 777                         iova = rte_mem_virt2iova(cur);
 778
 779                         iovas[cur_page] = iova;
 780                 }
 781
 782                 break;
 783         }
 784         /* if we couldn't allocate anything */
 785         if (iovas == NULL)
 786                 return -1;
 787
 788         param->addr = addr;
 789         param->len = mem_sz;
 790         param->pgsz = cur_pgsz;
 791         param->iova_table = iovas;
 792         param->iova_table_len = n_pages;
 793
 794         return 0;
 795 fail:
 796         if (iovas)
 797                 free(iovas);
 798         if (addr)
 799                 munmap(addr, mem_sz);
 800
 801         return -1;
 802 }
 803
 804 static int
 805 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 806 {
 807         struct extmem_param param;
 808         int socket_id, ret;
 809
 810         memset(&param, 0, sizeof(param));
 811
 812         /* check if our heap exists */
 813         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 814         if (socket_id < 0) {
 815                 /* create our heap */
 816                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 817                 if (ret < 0) {
 818                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 819                         return -1;
 820                 }
 821         }
 822
 823         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 824         if (ret < 0) {
 825                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 826                 return -1;
 827         }
 828
 829         /* we now have a valid memory area, so add it to heap */
 830         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 831                         param.addr, param.len, param.iova_table,
 832                         param.iova_table_len, param.pgsz);
 833
 834         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 835
 836         /* not needed any more */
 837         free(param.iova_table);
 838
 839         if (ret < 0) {
 840                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 841                 munmap(param.addr, param.len);
 842                 return -1;
 843         }
 844
 845         /* success */
 846
 847         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 848                         param.len >> 20);
 849
 850         return 0;
 851 }
 852 static void
 853 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 854              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 855 {
 856         uint16_t pid = 0;
 857         int ret;
 858
 859         RTE_ETH_FOREACH_DEV(pid) {
 860                 struct rte_eth_dev *dev =
 861                         &rte_eth_devices[pid];
 862
 863                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 864                                         memhdr->len);
 865                 if (ret) {
 866                         TESTPMD_LOG(DEBUG,
 867                                     "unable to DMA unmap addr 0x%p "
 868                                     "for device %s\n",
 869                                     memhdr->addr, dev->data->name);
 870                 }
 871         }
 872         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 873         if (ret) {
 874                 TESTPMD_LOG(DEBUG,
 875                             "unable to un-register addr 0x%p\n", memhdr->addr);
 876         }
 877 }
 878
 879 static void
 880 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 881            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 882 {
 883         uint16_t pid = 0;
 884         size_t page_size = sysconf(_SC_PAGESIZE);
 885         int ret;
 886
 887         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 888                                   page_size);
 889         if (ret) {
 890                 TESTPMD_LOG(DEBUG,
 891                             "unable to register addr 0x%p\n", memhdr->addr);
 892                 return;
 893         }
 894         RTE_ETH_FOREACH_DEV(pid) {
 895                 struct rte_eth_dev *dev =
 896                         &rte_eth_devices[pid];
 897
 898                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 899                                       memhdr->len);
 900                 if (ret) {
 901                         TESTPMD_LOG(DEBUG,
 902                                     "unable to DMA map addr 0x%p "
 903                                     "for device %s\n",
 904                                     memhdr->addr, dev->data->name);
 905                 }
 906         }
 907 }
 908 #endif
 909
 910 static unsigned int
 911 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 912             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 913 {
 914         struct rte_pktmbuf_extmem *xmem;
 915         unsigned int ext_num, zone_num, elt_num;
 916         uint16_t elt_size;
 917
 918         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 919         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 920         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 921
 922         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 923         if (xmem == NULL) {
 924                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 925                                  "external buffer descriptors\n");
 926                 *ext_mem = NULL;
 927                 return 0;
 928         }
 929         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 930                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 931                 const struct rte_memzone *mz;
 932                 char mz_name[RTE_MEMZONE_NAMESIZE];
 933                 int ret;
 934
 935                 ret = snprintf(mz_name, sizeof(mz_name),
 936                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 937                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 938                         errno = ENAMETOOLONG;
 939                         ext_num = 0;
 940                         break;
 941                 }
 942                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 943                                                  socket_id,
 944                                                  RTE_MEMZONE_IOVA_CONTIG |
 945                                                  RTE_MEMZONE_1GB |
 946                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 947                                                  EXTBUF_ZONE_SIZE);
 948                 if (mz == NULL) {
 949                         /*
 950                          * The caller exits on external buffer creation
 951                          * error, so there is no need to free memzones.
 952                          */
 953                         errno = ENOMEM;
 954                         ext_num = 0;
 955                         break;
 956                 }
 957                 xseg->buf_ptr = mz->addr;
 958                 xseg->buf_iova = mz->iova;
 959                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 960                 xseg->elt_size = elt_size;
 961         }
 962         if (ext_num == 0 && xmem != NULL) {
 963                 free(xmem);
 964                 xmem = NULL;
 965         }
 966         *ext_mem = xmem;
 967         return ext_num;
 968 }
 969
 970 /*
 971  * Configuration initialisation done once at init time.
 972  */
 973 static struct rte_mempool *
 974 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 975                  unsigned int socket_id, uint16_t size_idx)
 976 {
 977         char pool_name[RTE_MEMPOOL_NAMESIZE];
 978         struct rte_mempool *rte_mp = NULL;
 979 #ifndef RTE_EXEC_ENV_WINDOWS
 980         uint32_t mb_size;
 981
 982         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 983 #endif
 984         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 985
 986         TESTPMD_LOG(INFO,
 987                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 988                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 989
 990         switch (mp_alloc_type) {
 991         case MP_ALLOC_NATIVE:
 992                 {
 993                         /* wrapper to rte_mempool_create() */
 994                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 995                                         rte_mbuf_best_mempool_ops());
 996                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 997                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 998                         break;
 999                 }
1000 #ifndef RTE_EXEC_ENV_WINDOWS
1001         case MP_ALLOC_ANON:
1002                 {
1003                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1004                                 mb_size, (unsigned int) mb_mempool_cache,
1005                                 sizeof(struct rte_pktmbuf_pool_private),
1006                                 socket_id, mempool_flags);
1007                         if (rte_mp == NULL)
1008                                 goto err;
1009
1010                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1011                                 rte_mempool_free(rte_mp);
1012                                 rte_mp = NULL;
1013                                 goto err;
1014                         }
1015                         rte_pktmbuf_pool_init(rte_mp, NULL);
1016                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1017                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1018                         break;
1019                 }
1020         case MP_ALLOC_XMEM:
1021         case MP_ALLOC_XMEM_HUGE:
1022                 {
1023                         int heap_socket;
1024                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1025
1026                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1027                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1028
1029                         heap_socket =
1030                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1031                         if (heap_socket < 0)
1032                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1033
1034                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1035                                         rte_mbuf_best_mempool_ops());
1036                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1037                                         mb_mempool_cache, 0, mbuf_seg_size,
1038                                         heap_socket);
1039                         break;
1040                 }
1041 #endif
1042         case MP_ALLOC_XBUF:
1043                 {
1044                         struct rte_pktmbuf_extmem *ext_mem;
1045                         unsigned int ext_num;
1046
1047                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1048                                                socket_id, pool_name, &ext_mem);
1049                         if (ext_num == 0)
1050                                 rte_exit(EXIT_FAILURE,
1051                                          "Can't create pinned data buffers\n");
1052
1053                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1054                                         rte_mbuf_best_mempool_ops());
1055                         rte_mp = rte_pktmbuf_pool_create_extbuf
1056                                         (pool_name, nb_mbuf, mb_mempool_cache,
1057                                          0, mbuf_seg_size, socket_id,
1058                                          ext_mem, ext_num);
1059                         free(ext_mem);
1060                         break;
1061                 }
1062         default:
1063                 {
1064                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1065                 }
1066         }
1067
1068 #ifndef RTE_EXEC_ENV_WINDOWS
1069 err:
1070 #endif
1071         if (rte_mp == NULL) {
1072                 rte_exit(EXIT_FAILURE,
1073                         "Creation of mbuf pool for socket %u failed: %s\n",
1074                         socket_id, rte_strerror(rte_errno));
1075         } else if (verbose_level > 0) {
1076                 rte_mempool_dump(stdout, rte_mp);
1077         }
1078         return rte_mp;
1079 }
1080
1081 /*
1082  * Check given socket id is valid or not with NUMA mode,
1083  * if valid, return 0, else return -1
1084  */
1085 static int
1086 check_socket_id(const unsigned int socket_id)
1087 {
1088         static int warning_once = 0;
1089
1090         if (new_socket_id(socket_id)) {
1091                 if (!warning_once && numa_support)
1092                         printf("Warning: NUMA should be configured manually by"
1093                                " using --port-numa-config and"
1094                                " --ring-numa-config parameters along with"
1095                                " --numa.\n");
1096                 warning_once = 1;
1097                 return -1;
1098         }
1099         return 0;
1100 }
1101
1102 /*
1103  * Get the allowed maximum number of RX queues.
1104  * *pid return the port id which has minimal value of
1105  * max_rx_queues in all ports.
1106  */
1107 queueid_t
1108 get_allowed_max_nb_rxq(portid_t *pid)
1109 {
1110         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1111         bool max_rxq_valid = false;
1112         portid_t pi;
1113         struct rte_eth_dev_info dev_info;
1114
1115         RTE_ETH_FOREACH_DEV(pi) {
1116                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1117                         continue;
1118
1119                 max_rxq_valid = true;
1120                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1121                         allowed_max_rxq = dev_info.max_rx_queues;
1122                         *pid = pi;
1123                 }
1124         }
1125         return max_rxq_valid ? allowed_max_rxq : 0;
1126 }
1127
1128 /*
1129  * Check input rxq is valid or not.
1130  * If input rxq is not greater than any of maximum number
1131  * of RX queues of all ports, it is valid.
1132  * if valid, return 0, else return -1
1133  */
1134 int
1135 check_nb_rxq(queueid_t rxq)
1136 {
1137         queueid_t allowed_max_rxq;
1138         portid_t pid = 0;
1139
1140         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1141         if (rxq > allowed_max_rxq) {
1142                 printf("Fail: input rxq (%u) can't be greater "
1143                        "than max_rx_queues (%u) of port %u\n",
1144                        rxq,
1145                        allowed_max_rxq,
1146                        pid);
1147                 return -1;
1148         }
1149         return 0;
1150 }
1151
1152 /*
1153  * Get the allowed maximum number of TX queues.
1154  * *pid return the port id which has minimal value of
1155  * max_tx_queues in all ports.
1156  */
1157 queueid_t
1158 get_allowed_max_nb_txq(portid_t *pid)
1159 {
1160         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1161         bool max_txq_valid = false;
1162         portid_t pi;
1163         struct rte_eth_dev_info dev_info;
1164
1165         RTE_ETH_FOREACH_DEV(pi) {
1166                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1167                         continue;
1168
1169                 max_txq_valid = true;
1170                 if (dev_info.max_tx_queues < allowed_max_txq) {
1171                         allowed_max_txq = dev_info.max_tx_queues;
1172                         *pid = pi;
1173                 }
1174         }
1175         return max_txq_valid ? allowed_max_txq : 0;
1176 }
1177
1178 /*
1179  * Check input txq is valid or not.
1180  * If input txq is not greater than any of maximum number
1181  * of TX queues of all ports, it is valid.
1182  * if valid, return 0, else return -1
1183  */
1184 int
1185 check_nb_txq(queueid_t txq)
1186 {
1187         queueid_t allowed_max_txq;
1188         portid_t pid = 0;
1189
1190         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1191         if (txq > allowed_max_txq) {
1192                 printf("Fail: input txq (%u) can't be greater "
1193                        "than max_tx_queues (%u) of port %u\n",
1194                        txq,
1195                        allowed_max_txq,
1196                        pid);
1197                 return -1;
1198         }
1199         return 0;
1200 }
1201
1202 /*
1203  * Get the allowed maximum number of RXDs of every rx queue.
1204  * *pid return the port id which has minimal value of
1205  * max_rxd in all queues of all ports.
1206  */
1207 static uint16_t
1208 get_allowed_max_nb_rxd(portid_t *pid)
1209 {
1210         uint16_t allowed_max_rxd = UINT16_MAX;
1211         portid_t pi;
1212         struct rte_eth_dev_info dev_info;
1213
1214         RTE_ETH_FOREACH_DEV(pi) {
1215                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1216                         continue;
1217
1218                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1219                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1220                         *pid = pi;
1221                 }
1222         }
1223         return allowed_max_rxd;
1224 }
1225
1226 /*
1227  * Get the allowed minimal number of RXDs of every rx queue.
1228  * *pid return the port id which has minimal value of
1229  * min_rxd in all queues of all ports.
1230  */
1231 static uint16_t
1232 get_allowed_min_nb_rxd(portid_t *pid)
1233 {
1234         uint16_t allowed_min_rxd = 0;
1235         portid_t pi;
1236         struct rte_eth_dev_info dev_info;
1237
1238         RTE_ETH_FOREACH_DEV(pi) {
1239                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1240                         continue;
1241
1242                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1243                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1244                         *pid = pi;
1245                 }
1246         }
1247
1248         return allowed_min_rxd;
1249 }
1250
1251 /*
1252  * Check input rxd is valid or not.
1253  * If input rxd is not greater than any of maximum number
1254  * of RXDs of every Rx queues and is not less than any of
1255  * minimal number of RXDs of every Rx queues, it is valid.
1256  * if valid, return 0, else return -1
1257  */
1258 int
1259 check_nb_rxd(queueid_t rxd)
1260 {
1261         uint16_t allowed_max_rxd;
1262         uint16_t allowed_min_rxd;
1263         portid_t pid = 0;
1264
1265         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1266         if (rxd > allowed_max_rxd) {
1267                 printf("Fail: input rxd (%u) can't be greater "
1268                        "than max_rxds (%u) of port %u\n",
1269                        rxd,
1270                        allowed_max_rxd,
1271                        pid);
1272                 return -1;
1273         }
1274
1275         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1276         if (rxd < allowed_min_rxd) {
1277                 printf("Fail: input rxd (%u) can't be less "
1278                        "than min_rxds (%u) of port %u\n",
1279                        rxd,
1280                        allowed_min_rxd,
1281                        pid);
1282                 return -1;
1283         }
1284
1285         return 0;
1286 }
1287
1288 /*
1289  * Get the allowed maximum number of TXDs of every rx queues.
1290  * *pid return the port id which has minimal value of
1291  * max_txd in every tx queue.
1292  */
1293 static uint16_t
1294 get_allowed_max_nb_txd(portid_t *pid)
1295 {
1296         uint16_t allowed_max_txd = UINT16_MAX;
1297         portid_t pi;
1298         struct rte_eth_dev_info dev_info;
1299
1300         RTE_ETH_FOREACH_DEV(pi) {
1301                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1302                         continue;
1303
1304                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1305                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1306                         *pid = pi;
1307                 }
1308         }
1309         return allowed_max_txd;
1310 }
1311
1312 /*
1313  * Get the allowed maximum number of TXDs of every tx queues.
1314  * *pid return the port id which has minimal value of
1315  * min_txd in every tx queue.
1316  */
1317 static uint16_t
1318 get_allowed_min_nb_txd(portid_t *pid)
1319 {
1320         uint16_t allowed_min_txd = 0;
1321         portid_t pi;
1322         struct rte_eth_dev_info dev_info;
1323
1324         RTE_ETH_FOREACH_DEV(pi) {
1325                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1326                         continue;
1327
1328                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1329                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1330                         *pid = pi;
1331                 }
1332         }
1333
1334         return allowed_min_txd;
1335 }
1336
1337 /*
1338  * Check input txd is valid or not.
1339  * If input txd is not greater than any of maximum number
1340  * of TXDs of every Rx queues, it is valid.
1341  * if valid, return 0, else return -1
1342  */
1343 int
1344 check_nb_txd(queueid_t txd)
1345 {
1346         uint16_t allowed_max_txd;
1347         uint16_t allowed_min_txd;
1348         portid_t pid = 0;
1349
1350         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1351         if (txd > allowed_max_txd) {
1352                 printf("Fail: input txd (%u) can't be greater "
1353                        "than max_txds (%u) of port %u\n",
1354                        txd,
1355                        allowed_max_txd,
1356                        pid);
1357                 return -1;
1358         }
1359
1360         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1361         if (txd < allowed_min_txd) {
1362                 printf("Fail: input txd (%u) can't be less "
1363                        "than min_txds (%u) of port %u\n",
1364                        txd,
1365                        allowed_min_txd,
1366                        pid);
1367                 return -1;
1368         }
1369         return 0;
1370 }
1371
1372
1373 /*
1374  * Get the allowed maximum number of hairpin queues.
1375  * *pid return the port id which has minimal value of
1376  * max_hairpin_queues in all ports.
1377  */
1378 queueid_t
1379 get_allowed_max_nb_hairpinq(portid_t *pid)
1380 {
1381         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1382         portid_t pi;
1383         struct rte_eth_hairpin_cap cap;
1384
1385         RTE_ETH_FOREACH_DEV(pi) {
1386                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1387                         *pid = pi;
1388                         return 0;
1389                 }
1390                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1391                         allowed_max_hairpinq = cap.max_nb_queues;
1392                         *pid = pi;
1393                 }
1394         }
1395         return allowed_max_hairpinq;
1396 }
1397
1398 /*
1399  * Check input hairpin is valid or not.
1400  * If input hairpin is not greater than any of maximum number
1401  * of hairpin queues of all ports, it is valid.
1402  * if valid, return 0, else return -1
1403  */
1404 int
1405 check_nb_hairpinq(queueid_t hairpinq)
1406 {
1407         queueid_t allowed_max_hairpinq;
1408         portid_t pid = 0;
1409
1410         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1411         if (hairpinq > allowed_max_hairpinq) {
1412                 printf("Fail: input hairpin (%u) can't be greater "
1413                        "than max_hairpin_queues (%u) of port %u\n",
1414                        hairpinq, allowed_max_hairpinq, pid);
1415                 return -1;
1416         }
1417         return 0;
1418 }
1419
1420 static void
1421 init_config(void)
1422 {
1423         portid_t pid;
1424         struct rte_port *port;
1425         struct rte_mempool *mbp;
1426         unsigned int nb_mbuf_per_pool;
1427         lcoreid_t  lc_id;
1428         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1429         struct rte_gro_param gro_param;
1430         uint32_t gso_types;
1431         uint16_t data_size;
1432         bool warning = 0;
1433         int k;
1434         int ret;
1435
1436         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1437
1438         /* Configuration of logical cores. */
1439         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1440                                 sizeof(struct fwd_lcore *) * nb_lcores,
1441                                 RTE_CACHE_LINE_SIZE);
1442         if (fwd_lcores == NULL) {
1443                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1444                                                         "failed\n", nb_lcores);
1445         }
1446         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1447                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1448                                                sizeof(struct fwd_lcore),
1449                                                RTE_CACHE_LINE_SIZE);
1450                 if (fwd_lcores[lc_id] == NULL) {
1451                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1452                                                                 "failed\n");
1453                 }
1454                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1455         }
1456
1457         RTE_ETH_FOREACH_DEV(pid) {
1458                 port = &ports[pid];
1459                 /* Apply default TxRx configuration for all ports */
1460                 port->dev_conf.txmode = tx_mode;
1461                 port->dev_conf.rxmode = rx_mode;
1462
1463                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1464                 if (ret != 0)
1465                         rte_exit(EXIT_FAILURE,
1466                                  "rte_eth_dev_info_get() failed\n");
1467
1468                 ret = update_jumbo_frame_offload(pid);
1469                 if (ret != 0)
1470                         printf("Updating jumbo frame offload failed for port %u\n",
1471                                 pid);
1472
1473                 if (!(port->dev_info.tx_offload_capa &
1474                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1475                         port->dev_conf.txmode.offloads &=
1476                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1477                 if (numa_support) {
1478                         if (port_numa[pid] != NUMA_NO_CONFIG)
1479                                 port_per_socket[port_numa[pid]]++;
1480                         else {
1481                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1482
1483                                 /*
1484                                  * if socket_id is invalid,
1485                                  * set to the first available socket.
1486                                  */
1487                                 if (check_socket_id(socket_id) < 0)
1488                                         socket_id = socket_ids[0];
1489                                 port_per_socket[socket_id]++;
1490                         }
1491                 }
1492
1493                 /* Apply Rx offloads configuration */
1494                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1495                         port->rx_conf[k].offloads =
1496                                 port->dev_conf.rxmode.offloads;
1497                 /* Apply Tx offloads configuration */
1498                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1499                         port->tx_conf[k].offloads =
1500                                 port->dev_conf.txmode.offloads;
1501
1502                 if (eth_link_speed)
1503                         port->dev_conf.link_speeds = eth_link_speed;
1504
1505                 /* set flag to initialize port/queue */
1506                 port->need_reconfig = 1;
1507                 port->need_reconfig_queues = 1;
1508                 port->tx_metadata = 0;
1509
1510                 /* Check for maximum number of segments per MTU. Accordingly
1511                  * update the mbuf data size.
1512                  */
1513                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1514                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1515                         data_size = rx_mode.max_rx_pkt_len /
1516                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1517
1518                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1519                                                         mbuf_data_size[0]) {
1520                                 mbuf_data_size[0] = data_size +
1521                                                  RTE_PKTMBUF_HEADROOM;
1522                                 warning = 1;
1523                         }
1524                 }
1525         }
1526
1527         if (warning)
1528                 TESTPMD_LOG(WARNING,
1529                             "Configured mbuf size of the first segment %hu\n",
1530                             mbuf_data_size[0]);
1531         /*
1532          * Create pools of mbuf.
1533          * If NUMA support is disabled, create a single pool of mbuf in
1534          * socket 0 memory by default.
1535          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1536          *
1537          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1538          * nb_txd can be configured at run time.
1539          */
1540         if (param_total_num_mbufs)
1541                 nb_mbuf_per_pool = param_total_num_mbufs;
1542         else {
1543                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1544                         (nb_lcores * mb_mempool_cache) +
1545                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1546                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1547         }
1548
1549         if (numa_support) {
1550                 uint8_t i, j;
1551
1552                 for (i = 0; i < num_sockets; i++)
1553                         for (j = 0; j < mbuf_data_size_n; j++)
1554                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1555                                         mbuf_pool_create(mbuf_data_size[j],
1556                                                           nb_mbuf_per_pool,
1557                                                           socket_ids[i], j);
1558         } else {
1559                 uint8_t i;
1560
1561                 for (i = 0; i < mbuf_data_size_n; i++)
1562                         mempools[i] = mbuf_pool_create
1563                                         (mbuf_data_size[i],
1564                                          nb_mbuf_per_pool,
1565                                          socket_num == UMA_NO_CONFIG ?
1566                                          0 : socket_num, i);
1567         }
1568
1569         init_port_config();
1570
1571         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1572                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1573         /*
1574          * Records which Mbuf pool to use by each logical core, if needed.
1575          */
1576         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1577                 mbp = mbuf_pool_find(
1578                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1579
1580                 if (mbp == NULL)
1581                         mbp = mbuf_pool_find(0, 0);
1582                 fwd_lcores[lc_id]->mbp = mbp;
1583                 /* initialize GSO context */
1584                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1585                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1586                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1587                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1588                         RTE_ETHER_CRC_LEN;
1589                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1590         }
1591
1592         fwd_config_setup();
1593
1594         /* create a gro context for each lcore */
1595         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1596         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1597         gro_param.max_item_per_flow = MAX_PKT_BURST;
1598         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1599                 gro_param.socket_id = rte_lcore_to_socket_id(
1600                                 fwd_lcores_cpuids[lc_id]);
1601                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1602                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1603                         rte_exit(EXIT_FAILURE,
1604                                         "rte_gro_ctx_create() failed\n");
1605                 }
1606         }
1607 }
1608
1609
1610 void
1611 reconfig(portid_t new_port_id, unsigned socket_id)
1612 {
1613         struct rte_port *port;
1614         int ret;
1615
1616         /* Reconfiguration of Ethernet ports. */
1617         port = &ports[new_port_id];
1618
1619         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1620         if (ret != 0)
1621                 return;
1622
1623         /* set flag to initialize port/queue */
1624         port->need_reconfig = 1;
1625         port->need_reconfig_queues = 1;
1626         port->socket_id = socket_id;
1627
1628         init_port_config();
1629 }
1630
1631
1632 int
1633 init_fwd_streams(void)
1634 {
1635         portid_t pid;
1636         struct rte_port *port;
1637         streamid_t sm_id, nb_fwd_streams_new;
1638         queueid_t q;
1639
1640         /* set socket id according to numa or not */
1641         RTE_ETH_FOREACH_DEV(pid) {
1642                 port = &ports[pid];
1643                 if (nb_rxq > port->dev_info.max_rx_queues) {
1644                         printf("Fail: nb_rxq(%d) is greater than "
1645                                 "max_rx_queues(%d)\n", nb_rxq,
1646                                 port->dev_info.max_rx_queues);
1647                         return -1;
1648                 }
1649                 if (nb_txq > port->dev_info.max_tx_queues) {
1650                         printf("Fail: nb_txq(%d) is greater than "
1651                                 "max_tx_queues(%d)\n", nb_txq,
1652                                 port->dev_info.max_tx_queues);
1653                         return -1;
1654                 }
1655                 if (numa_support) {
1656                         if (port_numa[pid] != NUMA_NO_CONFIG)
1657                                 port->socket_id = port_numa[pid];
1658                         else {
1659                                 port->socket_id = rte_eth_dev_socket_id(pid);
1660
1661                                 /*
1662                                  * if socket_id is invalid,
1663                                  * set to the first available socket.
1664                                  */
1665                                 if (check_socket_id(port->socket_id) < 0)
1666                                         port->socket_id = socket_ids[0];
1667                         }
1668                 }
1669                 else {
1670                         if (socket_num == UMA_NO_CONFIG)
1671                                 port->socket_id = 0;
1672                         else
1673                                 port->socket_id = socket_num;
1674                 }
1675         }
1676
1677         q = RTE_MAX(nb_rxq, nb_txq);
1678         if (q == 0) {
1679                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1680                 return -1;
1681         }
1682         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1683         if (nb_fwd_streams_new == nb_fwd_streams)
1684                 return 0;
1685         /* clear the old */
1686         if (fwd_streams != NULL) {
1687                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1688                         if (fwd_streams[sm_id] == NULL)
1689                                 continue;
1690                         rte_free(fwd_streams[sm_id]);
1691                         fwd_streams[sm_id] = NULL;
1692                 }
1693                 rte_free(fwd_streams);
1694                 fwd_streams = NULL;
1695         }
1696
1697         /* init new */
1698         nb_fwd_streams = nb_fwd_streams_new;
1699         if (nb_fwd_streams) {
1700                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1701                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1702                         RTE_CACHE_LINE_SIZE);
1703                 if (fwd_streams == NULL)
1704                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1705                                  " (struct fwd_stream *)) failed\n",
1706                                  nb_fwd_streams);
1707
1708                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1709                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1710                                 " struct fwd_stream", sizeof(struct fwd_stream),
1711                                 RTE_CACHE_LINE_SIZE);
1712                         if (fwd_streams[sm_id] == NULL)
1713                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1714                                          "(struct fwd_stream) failed\n");
1715                 }
1716         }
1717
1718         return 0;
1719 }
1720
1721 static void
1722 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1723 {
1724         uint64_t total_burst, sburst;
1725         uint64_t nb_burst;
1726         uint64_t burst_stats[4];
1727         uint16_t pktnb_stats[4];
1728         uint16_t nb_pkt;
1729         int burst_percent[4], sburstp;
1730         int i;
1731
1732         /*
1733          * First compute the total number of packet bursts and the
1734          * two highest numbers of bursts of the same number of packets.
1735          */
1736         memset(&burst_stats, 0x0, sizeof(burst_stats));
1737         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1738
1739         /* Show stats for 0 burst size always */
1740         total_burst = pbs->pkt_burst_spread[0];
1741         burst_stats[0] = pbs->pkt_burst_spread[0];
1742         pktnb_stats[0] = 0;
1743
1744         /* Find the next 2 burst sizes with highest occurrences. */
1745         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1746                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1747
1748                 if (nb_burst == 0)
1749                         continue;
1750
1751                 total_burst += nb_burst;
1752
1753                 if (nb_burst > burst_stats[1]) {
1754                         burst_stats[2] = burst_stats[1];
1755                         pktnb_stats[2] = pktnb_stats[1];
1756                         burst_stats[1] = nb_burst;
1757                         pktnb_stats[1] = nb_pkt;
1758                 } else if (nb_burst > burst_stats[2]) {
1759                         burst_stats[2] = nb_burst;
1760                         pktnb_stats[2] = nb_pkt;
1761                 }
1762         }
1763         if (total_burst == 0)
1764                 return;
1765
1766         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1767         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1768                 if (i == 3) {
1769                         printf("%d%% of other]\n", 100 - sburstp);
1770                         return;
1771                 }
1772
1773                 sburst += burst_stats[i];
1774                 if (sburst == total_burst) {
1775                         printf("%d%% of %d pkts]\n",
1776                                 100 - sburstp, (int) pktnb_stats[i]);
1777                         return;
1778                 }
1779
1780                 burst_percent[i] =
1781                         (double)burst_stats[i] / total_burst * 100;
1782                 printf("%d%% of %d pkts + ",
1783                         burst_percent[i], (int) pktnb_stats[i]);
1784                 sburstp += burst_percent[i];
1785         }
1786 }
1787
1788 static void
1789 fwd_stream_stats_display(streamid_t stream_id)
1790 {
1791         struct fwd_stream *fs;
1792         static const char *fwd_top_stats_border = "-------";
1793
1794         fs = fwd_streams[stream_id];
1795         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1796             (fs->fwd_dropped == 0))
1797                 return;
1798         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1799                "TX Port=%2d/Queue=%2d %s\n",
1800                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1801                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1802         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1803                " TX-dropped: %-14"PRIu64,
1804                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1805
1806         /* if checksum mode */
1807         if (cur_fwd_eng == &csum_fwd_engine) {
1808                 printf("  RX- bad IP checksum: %-14"PRIu64
1809                        "  Rx- bad L4 checksum: %-14"PRIu64
1810                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1811                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1812                         fs->rx_bad_outer_l4_csum);
1813                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1814                         fs->rx_bad_outer_ip_csum);
1815         } else {
1816                 printf("\n");
1817         }
1818
1819         if (record_burst_stats) {
1820                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1821                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1822         }
1823 }
1824
1825 void
1826 fwd_stats_display(void)
1827 {
1828         static const char *fwd_stats_border = "----------------------";
1829         static const char *acc_stats_border = "+++++++++++++++";
1830         struct {
1831                 struct fwd_stream *rx_stream;
1832                 struct fwd_stream *tx_stream;
1833                 uint64_t tx_dropped;
1834                 uint64_t rx_bad_ip_csum;
1835                 uint64_t rx_bad_l4_csum;
1836                 uint64_t rx_bad_outer_l4_csum;
1837                 uint64_t rx_bad_outer_ip_csum;
1838         } ports_stats[RTE_MAX_ETHPORTS];
1839         uint64_t total_rx_dropped = 0;
1840         uint64_t total_tx_dropped = 0;
1841         uint64_t total_rx_nombuf = 0;
1842         struct rte_eth_stats stats;
1843         uint64_t fwd_cycles = 0;
1844         uint64_t total_recv = 0;
1845         uint64_t total_xmit = 0;
1846         struct rte_port *port;
1847         streamid_t sm_id;
1848         portid_t pt_id;
1849         int i;
1850
1851         memset(ports_stats, 0, sizeof(ports_stats));
1852
1853         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1854                 struct fwd_stream *fs = fwd_streams[sm_id];
1855
1856                 if (cur_fwd_config.nb_fwd_streams >
1857                     cur_fwd_config.nb_fwd_ports) {
1858                         fwd_stream_stats_display(sm_id);
1859                 } else {
1860                         ports_stats[fs->tx_port].tx_stream = fs;
1861                         ports_stats[fs->rx_port].rx_stream = fs;
1862                 }
1863
1864                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1865
1866                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1867                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1868                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1869                                 fs->rx_bad_outer_l4_csum;
1870                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
1871                                 fs->rx_bad_outer_ip_csum;
1872
1873                 if (record_core_cycles)
1874                         fwd_cycles += fs->core_cycles;
1875         }
1876         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1877                 pt_id = fwd_ports_ids[i];
1878                 port = &ports[pt_id];
1879
1880                 rte_eth_stats_get(pt_id, &stats);
1881                 stats.ipackets -= port->stats.ipackets;
1882                 stats.opackets -= port->stats.opackets;
1883                 stats.ibytes -= port->stats.ibytes;
1884                 stats.obytes -= port->stats.obytes;
1885                 stats.imissed -= port->stats.imissed;
1886                 stats.oerrors -= port->stats.oerrors;
1887                 stats.rx_nombuf -= port->stats.rx_nombuf;
1888
1889                 total_recv += stats.ipackets;
1890                 total_xmit += stats.opackets;
1891                 total_rx_dropped += stats.imissed;
1892                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1893                 total_tx_dropped += stats.oerrors;
1894                 total_rx_nombuf  += stats.rx_nombuf;
1895
1896                 printf("\n  %s Forward statistics for port %-2d %s\n",
1897                        fwd_stats_border, pt_id, fwd_stats_border);
1898
1899                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1900                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1901                        stats.ipackets + stats.imissed);
1902
1903                 if (cur_fwd_eng == &csum_fwd_engine) {
1904                         printf("  Bad-ipcsum: %-14"PRIu64
1905                                " Bad-l4csum: %-14"PRIu64
1906                                "Bad-outer-l4csum: %-14"PRIu64"\n",
1907                                ports_stats[pt_id].rx_bad_ip_csum,
1908                                ports_stats[pt_id].rx_bad_l4_csum,
1909                                ports_stats[pt_id].rx_bad_outer_l4_csum);
1910                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
1911                                ports_stats[pt_id].rx_bad_outer_ip_csum);
1912                 }
1913                 if (stats.ierrors + stats.rx_nombuf > 0) {
1914                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
1915                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1916                 }
1917
1918                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1919                        "TX-total: %-"PRIu64"\n",
1920                        stats.opackets, ports_stats[pt_id].tx_dropped,
1921                        stats.opackets + ports_stats[pt_id].tx_dropped);
1922
1923                 if (record_burst_stats) {
1924                         if (ports_stats[pt_id].rx_stream)
1925                                 pkt_burst_stats_display("RX",
1926                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1927                         if (ports_stats[pt_id].tx_stream)
1928                                 pkt_burst_stats_display("TX",
1929                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1930                 }
1931
1932                 printf("  %s--------------------------------%s\n",
1933                        fwd_stats_border, fwd_stats_border);
1934         }
1935
1936         printf("\n  %s Accumulated forward statistics for all ports"
1937                "%s\n",
1938                acc_stats_border, acc_stats_border);
1939         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1940                "%-"PRIu64"\n"
1941                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1942                "%-"PRIu64"\n",
1943                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1944                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1945         if (total_rx_nombuf > 0)
1946                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1947         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1948                "%s\n",
1949                acc_stats_border, acc_stats_border);
1950         if (record_core_cycles) {
1951 #define CYC_PER_MHZ 1E6
1952                 if (total_recv > 0 || total_xmit > 0) {
1953                         uint64_t total_pkts = 0;
1954                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1955                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1956                                 total_pkts = total_xmit;
1957                         else
1958                                 total_pkts = total_recv;
1959
1960                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1961                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1962                                " MHz Clock\n",
1963                                (double) fwd_cycles / total_pkts,
1964                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1965                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1966                 }
1967         }
1968 }
1969
1970 void
1971 fwd_stats_reset(void)
1972 {
1973         streamid_t sm_id;
1974         portid_t pt_id;
1975         int i;
1976
1977         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1978                 pt_id = fwd_ports_ids[i];
1979                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1980         }
1981         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1982                 struct fwd_stream *fs = fwd_streams[sm_id];
1983
1984                 fs->rx_packets = 0;
1985                 fs->tx_packets = 0;
1986                 fs->fwd_dropped = 0;
1987                 fs->rx_bad_ip_csum = 0;
1988                 fs->rx_bad_l4_csum = 0;
1989                 fs->rx_bad_outer_l4_csum = 0;
1990                 fs->rx_bad_outer_ip_csum = 0;
1991
1992                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1993                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1994                 fs->core_cycles = 0;
1995         }
1996 }
1997
1998 static void
1999 flush_fwd_rx_queues(void)
2000 {
2001         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2002         portid_t  rxp;
2003         portid_t port_id;
2004         queueid_t rxq;
2005         uint16_t  nb_rx;
2006         uint16_t  i;
2007         uint8_t   j;
2008         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2009         uint64_t timer_period;
2010
2011         /* convert to number of cycles */
2012         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2013
2014         for (j = 0; j < 2; j++) {
2015                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2016                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2017                                 port_id = fwd_ports_ids[rxp];
2018                                 /**
2019                                 * testpmd can stuck in the below do while loop
2020                                 * if rte_eth_rx_burst() always returns nonzero
2021                                 * packets. So timer is added to exit this loop
2022                                 * after 1sec timer expiry.
2023                                 */
2024                                 prev_tsc = rte_rdtsc();
2025                                 do {
2026                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2027                                                 pkts_burst, MAX_PKT_BURST);
2028                                         for (i = 0; i < nb_rx; i++)
2029                                                 rte_pktmbuf_free(pkts_burst[i]);
2030
2031                                         cur_tsc = rte_rdtsc();
2032                                         diff_tsc = cur_tsc - prev_tsc;
2033                                         timer_tsc += diff_tsc;
2034                                 } while ((nb_rx > 0) &&
2035                                         (timer_tsc < timer_period));
2036                                 timer_tsc = 0;
2037                         }
2038                 }
2039                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2040         }
2041 }
2042
2043 static void
2044 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2045 {
2046         struct fwd_stream **fsm;
2047         streamid_t nb_fs;
2048         streamid_t sm_id;
2049 #ifdef RTE_LIB_BITRATESTATS
2050         uint64_t tics_per_1sec;
2051         uint64_t tics_datum;
2052         uint64_t tics_current;
2053         uint16_t i, cnt_ports;
2054
2055         cnt_ports = nb_ports;
2056         tics_datum = rte_rdtsc();
2057         tics_per_1sec = rte_get_timer_hz();
2058 #endif
2059         fsm = &fwd_streams[fc->stream_idx];
2060         nb_fs = fc->stream_nb;
2061         do {
2062                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2063                         (*pkt_fwd)(fsm[sm_id]);
2064 #ifdef RTE_LIB_BITRATESTATS
2065                 if (bitrate_enabled != 0 &&
2066                                 bitrate_lcore_id == rte_lcore_id()) {
2067                         tics_current = rte_rdtsc();
2068                         if (tics_current - tics_datum >= tics_per_1sec) {
2069                                 /* Periodic bitrate calculation */
2070                                 for (i = 0; i < cnt_ports; i++)
2071                                         rte_stats_bitrate_calc(bitrate_data,
2072                                                 ports_ids[i]);
2073                                 tics_datum = tics_current;
2074                         }
2075                 }
2076 #endif
2077 #ifdef RTE_LIB_LATENCYSTATS
2078                 if (latencystats_enabled != 0 &&
2079                                 latencystats_lcore_id == rte_lcore_id())
2080                         rte_latencystats_update();
2081 #endif
2082
2083         } while (! fc->stopped);
2084 }
2085
2086 static int
2087 start_pkt_forward_on_core(void *fwd_arg)
2088 {
2089         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2090                              cur_fwd_config.fwd_eng->packet_fwd);
2091         return 0;
2092 }
2093
2094 /*
2095  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2096  * Used to start communication flows in network loopback test configurations.
2097  */
2098 static int
2099 run_one_txonly_burst_on_core(void *fwd_arg)
2100 {
2101         struct fwd_lcore *fwd_lc;
2102         struct fwd_lcore tmp_lcore;
2103
2104         fwd_lc = (struct fwd_lcore *) fwd_arg;
2105         tmp_lcore = *fwd_lc;
2106         tmp_lcore.stopped = 1;
2107         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2108         return 0;
2109 }
2110
2111 /*
2112  * Launch packet forwarding:
2113  *     - Setup per-port forwarding context.
2114  *     - launch logical cores with their forwarding configuration.
2115  */
2116 static void
2117 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2118 {
2119         port_fwd_begin_t port_fwd_begin;
2120         unsigned int i;
2121         unsigned int lc_id;
2122         int diag;
2123
2124         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2125         if (port_fwd_begin != NULL) {
2126                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2127                         (*port_fwd_begin)(fwd_ports_ids[i]);
2128         }
2129         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2130                 lc_id = fwd_lcores_cpuids[i];
2131                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2132                         fwd_lcores[i]->stopped = 0;
2133                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2134                                                      fwd_lcores[i], lc_id);
2135                         if (diag != 0)
2136                                 printf("launch lcore %u failed - diag=%d\n",
2137                                        lc_id, diag);
2138                 }
2139         }
2140 }
2141
2142 /*
2143  * Launch packet forwarding configuration.
2144  */
2145 void
2146 start_packet_forwarding(int with_tx_first)
2147 {
2148         port_fwd_begin_t port_fwd_begin;
2149         port_fwd_end_t  port_fwd_end;
2150         unsigned int i;
2151
2152         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2153                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2154
2155         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2156                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2157
2158         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2159                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2160                 (!nb_rxq || !nb_txq))
2161                 rte_exit(EXIT_FAILURE,
2162                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2163                         cur_fwd_eng->fwd_mode_name);
2164
2165         if (all_ports_started() == 0) {
2166                 printf("Not all ports were started\n");
2167                 return;
2168         }
2169         if (test_done == 0) {
2170                 printf("Packet forwarding already started\n");
2171                 return;
2172         }
2173         test_done = 0;
2174
2175         fwd_config_setup();
2176
2177         if(!no_flush_rx)
2178                 flush_fwd_rx_queues();
2179
2180         pkt_fwd_config_display(&cur_fwd_config);
2181         rxtx_config_display();
2182
2183         fwd_stats_reset();
2184         if (with_tx_first) {
2185                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2186                 if (port_fwd_begin != NULL) {
2187                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2188                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2189                 }
2190                 while (with_tx_first--) {
2191                         launch_packet_forwarding(
2192                                         run_one_txonly_burst_on_core);
2193                         rte_eal_mp_wait_lcore();
2194                 }
2195                 port_fwd_end = tx_only_engine.port_fwd_end;
2196                 if (port_fwd_end != NULL) {
2197                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2198                                 (*port_fwd_end)(fwd_ports_ids[i]);
2199                 }
2200         }
2201         launch_packet_forwarding(start_pkt_forward_on_core);
2202 }
2203
2204 void
2205 stop_packet_forwarding(void)
2206 {
2207         port_fwd_end_t port_fwd_end;
2208         lcoreid_t lc_id;
2209         portid_t pt_id;
2210         int i;
2211
2212         if (test_done) {
2213                 printf("Packet forwarding not started\n");
2214                 return;
2215         }
2216         printf("Telling cores to stop...");
2217         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2218                 fwd_lcores[lc_id]->stopped = 1;
2219         printf("\nWaiting for lcores to finish...\n");
2220         rte_eal_mp_wait_lcore();
2221         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2222         if (port_fwd_end != NULL) {
2223                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2224                         pt_id = fwd_ports_ids[i];
2225                         (*port_fwd_end)(pt_id);
2226                 }
2227         }
2228
2229         fwd_stats_display();
2230
2231         printf("\nDone.\n");
2232         test_done = 1;
2233 }
2234
2235 void
2236 dev_set_link_up(portid_t pid)
2237 {
2238         if (rte_eth_dev_set_link_up(pid) < 0)
2239                 printf("\nSet link up fail.\n");
2240 }
2241
2242 void
2243 dev_set_link_down(portid_t pid)
2244 {
2245         if (rte_eth_dev_set_link_down(pid) < 0)
2246                 printf("\nSet link down fail.\n");
2247 }
2248
2249 static int
2250 all_ports_started(void)
2251 {
2252         portid_t pi;
2253         struct rte_port *port;
2254
2255         RTE_ETH_FOREACH_DEV(pi) {
2256                 port = &ports[pi];
2257                 /* Check if there is a port which is not started */
2258                 if ((port->port_status != RTE_PORT_STARTED) &&
2259                         (port->slave_flag == 0))
2260                         return 0;
2261         }
2262
2263         /* No port is not started */
2264         return 1;
2265 }
2266
2267 int
2268 port_is_stopped(portid_t port_id)
2269 {
2270         struct rte_port *port = &ports[port_id];
2271
2272         if ((port->port_status != RTE_PORT_STOPPED) &&
2273             (port->slave_flag == 0))
2274                 return 0;
2275         return 1;
2276 }
2277
2278 int
2279 all_ports_stopped(void)
2280 {
2281         portid_t pi;
2282
2283         RTE_ETH_FOREACH_DEV(pi) {
2284                 if (!port_is_stopped(pi))
2285                         return 0;
2286         }
2287
2288         return 1;
2289 }
2290
2291 int
2292 port_is_started(portid_t port_id)
2293 {
2294         if (port_id_is_invalid(port_id, ENABLED_WARN))
2295                 return 0;
2296
2297         if (ports[port_id].port_status != RTE_PORT_STARTED)
2298                 return 0;
2299
2300         return 1;
2301 }
2302
2303 /* Configure the Rx and Tx hairpin queues for the selected port. */
2304 static int
2305 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2306 {
2307         queueid_t qi;
2308         struct rte_eth_hairpin_conf hairpin_conf = {
2309                 .peer_count = 1,
2310         };
2311         int i;
2312         int diag;
2313         struct rte_port *port = &ports[pi];
2314         uint16_t peer_rx_port = pi;
2315         uint16_t peer_tx_port = pi;
2316         uint32_t manual = 1;
2317         uint32_t tx_exp = hairpin_mode & 0x10;
2318
2319         if (!(hairpin_mode & 0xf)) {
2320                 peer_rx_port = pi;
2321                 peer_tx_port = pi;
2322                 manual = 0;
2323         } else if (hairpin_mode & 0x1) {
2324                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2325                                                        RTE_ETH_DEV_NO_OWNER);
2326                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2327                         peer_tx_port = rte_eth_find_next_owned_by(0,
2328                                                 RTE_ETH_DEV_NO_OWNER);
2329                 if (p_pi != RTE_MAX_ETHPORTS) {
2330                         peer_rx_port = p_pi;
2331                 } else {
2332                         uint16_t next_pi;
2333
2334                         /* Last port will be the peer RX port of the first. */
2335                         RTE_ETH_FOREACH_DEV(next_pi)
2336                                 peer_rx_port = next_pi;
2337                 }
2338                 manual = 1;
2339         } else if (hairpin_mode & 0x2) {
2340                 if (cnt_pi & 0x1) {
2341                         peer_rx_port = p_pi;
2342                 } else {
2343                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2344                                                 RTE_ETH_DEV_NO_OWNER);
2345                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2346                                 peer_rx_port = pi;
2347                 }
2348                 peer_tx_port = peer_rx_port;
2349                 manual = 1;
2350         }
2351
2352         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2353                 hairpin_conf.peers[0].port = peer_rx_port;
2354                 hairpin_conf.peers[0].queue = i + nb_rxq;
2355                 hairpin_conf.manual_bind = !!manual;
2356                 hairpin_conf.tx_explicit = !!tx_exp;
2357                 diag = rte_eth_tx_hairpin_queue_setup
2358                         (pi, qi, nb_txd, &hairpin_conf);
2359                 i++;
2360                 if (diag == 0)
2361                         continue;
2362
2363                 /* Fail to setup rx queue, return */
2364                 if (rte_atomic16_cmpset(&(port->port_status),
2365                                         RTE_PORT_HANDLING,
2366                                         RTE_PORT_STOPPED) == 0)
2367                         printf("Port %d can not be set back "
2368                                         "to stopped\n", pi);
2369                 printf("Fail to configure port %d hairpin "
2370                                 "queues\n", pi);
2371                 /* try to reconfigure queues next time */
2372                 port->need_reconfig_queues = 1;
2373                 return -1;
2374         }
2375         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2376                 hairpin_conf.peers[0].port = peer_tx_port;
2377                 hairpin_conf.peers[0].queue = i + nb_txq;
2378                 hairpin_conf.manual_bind = !!manual;
2379                 hairpin_conf.tx_explicit = !!tx_exp;
2380                 diag = rte_eth_rx_hairpin_queue_setup
2381                         (pi, qi, nb_rxd, &hairpin_conf);
2382                 i++;
2383                 if (diag == 0)
2384                         continue;
2385
2386                 /* Fail to setup rx queue, return */
2387                 if (rte_atomic16_cmpset(&(port->port_status),
2388                                         RTE_PORT_HANDLING,
2389                                         RTE_PORT_STOPPED) == 0)
2390                         printf("Port %d can not be set back "
2391                                         "to stopped\n", pi);
2392                 printf("Fail to configure port %d hairpin "
2393                                 "queues\n", pi);
2394                 /* try to reconfigure queues next time */
2395                 port->need_reconfig_queues = 1;
2396                 return -1;
2397         }
2398         return 0;
2399 }
2400
2401 /* Configure the Rx with optional split. */
2402 int
2403 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2404                uint16_t nb_rx_desc, unsigned int socket_id,
2405                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2406 {
2407         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2408         unsigned int i, mp_n;
2409         int ret;
2410
2411         if (rx_pkt_nb_segs <= 1 ||
2412             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2413                 rx_conf->rx_seg = NULL;
2414                 rx_conf->rx_nseg = 0;
2415                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2416                                              nb_rx_desc, socket_id,
2417                                              rx_conf, mp);
2418                 return ret;
2419         }
2420         for (i = 0; i < rx_pkt_nb_segs; i++) {
2421                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2422                 struct rte_mempool *mpx;
2423                 /*
2424                  * Use last valid pool for the segments with number
2425                  * exceeding the pool index.
2426                  */
2427                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2428                 mpx = mbuf_pool_find(socket_id, mp_n);
2429                 /* Handle zero as mbuf data buffer size. */
2430                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2431                                    rx_pkt_seg_lengths[i] :
2432                                    mbuf_data_size[mp_n];
2433                 rx_seg->offset = i < rx_pkt_nb_offs ?
2434                                    rx_pkt_seg_offsets[i] : 0;
2435                 rx_seg->mp = mpx ? mpx : mp;
2436         }
2437         rx_conf->rx_nseg = rx_pkt_nb_segs;
2438         rx_conf->rx_seg = rx_useg;
2439         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2440                                     socket_id, rx_conf, NULL);
2441         rx_conf->rx_seg = NULL;
2442         rx_conf->rx_nseg = 0;
2443         return ret;
2444 }
2445
2446 int
2447 start_port(portid_t pid)
2448 {
2449         int diag, need_check_link_status = -1;
2450         portid_t pi;
2451         portid_t p_pi = RTE_MAX_ETHPORTS;
2452         portid_t pl[RTE_MAX_ETHPORTS];
2453         portid_t peer_pl[RTE_MAX_ETHPORTS];
2454         uint16_t cnt_pi = 0;
2455         uint16_t cfg_pi = 0;
2456         int peer_pi;
2457         queueid_t qi;
2458         struct rte_port *port;
2459         struct rte_ether_addr mac_addr;
2460         struct rte_eth_hairpin_cap cap;
2461
2462         if (port_id_is_invalid(pid, ENABLED_WARN))
2463                 return 0;
2464
2465         RTE_ETH_FOREACH_DEV(pi) {
2466                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2467                         continue;
2468
2469                 need_check_link_status = 0;
2470                 port = &ports[pi];
2471                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2472                                                  RTE_PORT_HANDLING) == 0) {
2473                         printf("Port %d is now not stopped\n", pi);
2474                         continue;
2475                 }
2476
2477                 if (port->need_reconfig > 0) {
2478                         port->need_reconfig = 0;
2479
2480                         if (flow_isolate_all) {
2481                                 int ret = port_flow_isolate(pi, 1);
2482                                 if (ret) {
2483                                         printf("Failed to apply isolated"
2484                                                " mode on port %d\n", pi);
2485                                         return -1;
2486                                 }
2487                         }
2488                         configure_rxtx_dump_callbacks(0);
2489                         printf("Configuring Port %d (socket %u)\n", pi,
2490                                         port->socket_id);
2491                         if (nb_hairpinq > 0 &&
2492                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2493                                 printf("Port %d doesn't support hairpin "
2494                                        "queues\n", pi);
2495                                 return -1;
2496                         }
2497                         /* configure port */
2498                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2499                                                      nb_txq + nb_hairpinq,
2500                                                      &(port->dev_conf));
2501                         if (diag != 0) {
2502                                 if (rte_atomic16_cmpset(&(port->port_status),
2503                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2504                                         printf("Port %d can not be set back "
2505                                                         "to stopped\n", pi);
2506                                 printf("Fail to configure port %d\n", pi);
2507                                 /* try to reconfigure port next time */
2508                                 port->need_reconfig = 1;
2509                                 return -1;
2510                         }
2511                 }
2512                 if (port->need_reconfig_queues > 0) {
2513                         port->need_reconfig_queues = 0;
2514                         /* setup tx queues */
2515                         for (qi = 0; qi < nb_txq; qi++) {
2516                                 if ((numa_support) &&
2517                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2518                                         diag = rte_eth_tx_queue_setup(pi, qi,
2519                                                 port->nb_tx_desc[qi],
2520                                                 txring_numa[pi],
2521                                                 &(port->tx_conf[qi]));
2522                                 else
2523                                         diag = rte_eth_tx_queue_setup(pi, qi,
2524                                                 port->nb_tx_desc[qi],
2525                                                 port->socket_id,
2526                                                 &(port->tx_conf[qi]));
2527
2528                                 if (diag == 0)
2529                                         continue;
2530
2531                                 /* Fail to setup tx queue, return */
2532                                 if (rte_atomic16_cmpset(&(port->port_status),
2533                                                         RTE_PORT_HANDLING,
2534                                                         RTE_PORT_STOPPED) == 0)
2535                                         printf("Port %d can not be set back "
2536                                                         "to stopped\n", pi);
2537                                 printf("Fail to configure port %d tx queues\n",
2538                                        pi);
2539                                 /* try to reconfigure queues next time */
2540                                 port->need_reconfig_queues = 1;
2541                                 return -1;
2542                         }
2543                         for (qi = 0; qi < nb_rxq; qi++) {
2544                                 /* setup rx queues */
2545                                 if ((numa_support) &&
2546                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2547                                         struct rte_mempool * mp =
2548                                                 mbuf_pool_find
2549                                                         (rxring_numa[pi], 0);
2550                                         if (mp == NULL) {
2551                                                 printf("Failed to setup RX queue:"
2552                                                         "No mempool allocation"
2553                                                         " on the socket %d\n",
2554                                                         rxring_numa[pi]);
2555                                                 return -1;
2556                                         }
2557
2558                                         diag = rx_queue_setup(pi, qi,
2559                                              port->nb_rx_desc[qi],
2560                                              rxring_numa[pi],
2561                                              &(port->rx_conf[qi]),
2562                                              mp);
2563                                 } else {
2564                                         struct rte_mempool *mp =
2565                                                 mbuf_pool_find
2566                                                         (port->socket_id, 0);
2567                                         if (mp == NULL) {
2568                                                 printf("Failed to setup RX queue:"
2569                                                         "No mempool allocation"
2570                                                         " on the socket %d\n",
2571                                                         port->socket_id);
2572                                                 return -1;
2573                                         }
2574                                         diag = rx_queue_setup(pi, qi,
2575                                              port->nb_rx_desc[qi],
2576                                              port->socket_id,
2577                                              &(port->rx_conf[qi]),
2578                                              mp);
2579                                 }
2580                                 if (diag == 0)
2581                                         continue;
2582
2583                                 /* Fail to setup rx queue, return */
2584                                 if (rte_atomic16_cmpset(&(port->port_status),
2585                                                         RTE_PORT_HANDLING,
2586                                                         RTE_PORT_STOPPED) == 0)
2587                                         printf("Port %d can not be set back "
2588                                                         "to stopped\n", pi);
2589                                 printf("Fail to configure port %d rx queues\n",
2590                                        pi);
2591                                 /* try to reconfigure queues next time */
2592                                 port->need_reconfig_queues = 1;
2593                                 return -1;
2594                         }
2595                         /* setup hairpin queues */
2596                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2597                                 return -1;
2598                 }
2599                 configure_rxtx_dump_callbacks(verbose_level);
2600                 if (clear_ptypes) {
2601                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2602                                         NULL, 0);
2603                         if (diag < 0)
2604                                 printf(
2605                                 "Port %d: Failed to disable Ptype parsing\n",
2606                                 pi);
2607                 }
2608
2609                 p_pi = pi;
2610                 cnt_pi++;
2611
2612                 /* start port */
2613                 diag = rte_eth_dev_start(pi);
2614                 if (diag < 0) {
2615                         printf("Fail to start port %d: %s\n", pi,
2616                                rte_strerror(-diag));
2617
2618                         /* Fail to setup rx queue, return */
2619                         if (rte_atomic16_cmpset(&(port->port_status),
2620                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2621                                 printf("Port %d can not be set back to "
2622                                                         "stopped\n", pi);
2623                         continue;
2624                 }
2625
2626                 if (rte_atomic16_cmpset(&(port->port_status),
2627                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2628                         printf("Port %d can not be set into started\n", pi);
2629
2630                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2631                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2632                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2633                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2634                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2635
2636                 /* at least one port started, need checking link status */
2637                 need_check_link_status = 1;
2638
2639                 pl[cfg_pi++] = pi;
2640         }
2641
2642         if (need_check_link_status == 1 && !no_link_check)
2643                 check_all_ports_link_status(RTE_PORT_ALL);
2644         else if (need_check_link_status == 0)
2645                 printf("Please stop the ports first\n");
2646
2647         if (hairpin_mode & 0xf) {
2648                 uint16_t i;
2649                 int j;
2650
2651                 /* bind all started hairpin ports */
2652                 for (i = 0; i < cfg_pi; i++) {
2653                         pi = pl[i];
2654                         /* bind current Tx to all peer Rx */
2655                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2656                                                         RTE_MAX_ETHPORTS, 1);
2657                         if (peer_pi < 0)
2658                                 return peer_pi;
2659                         for (j = 0; j < peer_pi; j++) {
2660                                 if (!port_is_started(peer_pl[j]))
2661                                         continue;
2662                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2663                                 if (diag < 0) {
2664                                         printf("Error during binding hairpin"
2665                                                " Tx port %u to %u: %s\n",
2666                                                pi, peer_pl[j],
2667                                                rte_strerror(-diag));
2668                                         return -1;
2669                                 }
2670                         }
2671                         /* bind all peer Tx to current Rx */
2672                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2673                                                         RTE_MAX_ETHPORTS, 0);
2674                         if (peer_pi < 0)
2675                                 return peer_pi;
2676                         for (j = 0; j < peer_pi; j++) {
2677                                 if (!port_is_started(peer_pl[j]))
2678                                         continue;
2679                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2680                                 if (diag < 0) {
2681                                         printf("Error during binding hairpin"
2682                                                " Tx port %u to %u: %s\n",
2683                                                peer_pl[j], pi,
2684                                                rte_strerror(-diag));
2685                                         return -1;
2686                                 }
2687                         }
2688                 }
2689         }
2690
2691         printf("Done\n");
2692         return 0;
2693 }
2694
2695 void
2696 stop_port(portid_t pid)
2697 {
2698         portid_t pi;
2699         struct rte_port *port;
2700         int need_check_link_status = 0;
2701         portid_t peer_pl[RTE_MAX_ETHPORTS];
2702         int peer_pi;
2703
2704         if (port_id_is_invalid(pid, ENABLED_WARN))
2705                 return;
2706
2707         printf("Stopping ports...\n");
2708
2709         RTE_ETH_FOREACH_DEV(pi) {
2710                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2711                         continue;
2712
2713                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2714                         printf("Please remove port %d from forwarding configuration.\n", pi);
2715                         continue;
2716                 }
2717
2718                 if (port_is_bonding_slave(pi)) {
2719                         printf("Please remove port %d from bonded device.\n", pi);
2720                         continue;
2721                 }
2722
2723                 port = &ports[pi];
2724                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2725                                                 RTE_PORT_HANDLING) == 0)
2726                         continue;
2727
2728                 if (hairpin_mode & 0xf) {
2729                         int j;
2730
2731                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2732                         /* unbind all peer Tx from current Rx */
2733                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2734                                                         RTE_MAX_ETHPORTS, 0);
2735                         if (peer_pi < 0)
2736                                 continue;
2737                         for (j = 0; j < peer_pi; j++) {
2738                                 if (!port_is_started(peer_pl[j]))
2739                                         continue;
2740                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2741                         }
2742                 }
2743
2744                 if (port->flow_list)
2745                         port_flow_flush(pi);
2746
2747                 if (rte_eth_dev_stop(pi) != 0)
2748                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2749                                 pi);
2750
2751                 if (rte_atomic16_cmpset(&(port->port_status),
2752                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2753                         printf("Port %d can not be set into stopped\n", pi);
2754                 need_check_link_status = 1;
2755         }
2756         if (need_check_link_status && !no_link_check)
2757                 check_all_ports_link_status(RTE_PORT_ALL);
2758
2759         printf("Done\n");
2760 }
2761
2762 static void
2763 remove_invalid_ports_in(portid_t *array, portid_t *total)
2764 {
2765         portid_t i;
2766         portid_t new_total = 0;
2767
2768         for (i = 0; i < *total; i++)
2769                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2770                         array[new_total] = array[i];
2771                         new_total++;
2772                 }
2773         *total = new_total;
2774 }
2775
2776 static void
2777 remove_invalid_ports(void)
2778 {
2779         remove_invalid_ports_in(ports_ids, &nb_ports);
2780         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2781         nb_cfg_ports = nb_fwd_ports;
2782 }
2783
2784 void
2785 close_port(portid_t pid)
2786 {
2787         portid_t pi;
2788         struct rte_port *port;
2789
2790         if (port_id_is_invalid(pid, ENABLED_WARN))
2791                 return;
2792
2793         printf("Closing ports...\n");
2794
2795         RTE_ETH_FOREACH_DEV(pi) {
2796                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2797                         continue;
2798
2799                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2800                         printf("Please remove port %d from forwarding configuration.\n", pi);
2801                         continue;
2802                 }
2803
2804                 if (port_is_bonding_slave(pi)) {
2805                         printf("Please remove port %d from bonded device.\n", pi);
2806                         continue;
2807                 }
2808
2809                 port = &ports[pi];
2810                 if (rte_atomic16_cmpset(&(port->port_status),
2811                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2812                         printf("Port %d is already closed\n", pi);
2813                         continue;
2814                 }
2815
2816                 port_flow_flush(pi);
2817                 rte_eth_dev_close(pi);
2818         }
2819
2820         remove_invalid_ports();
2821         printf("Done\n");
2822 }
2823
2824 void
2825 reset_port(portid_t pid)
2826 {
2827         int diag;
2828         portid_t pi;
2829         struct rte_port *port;
2830
2831         if (port_id_is_invalid(pid, ENABLED_WARN))
2832                 return;
2833
2834         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2835                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2836                 printf("Can not reset port(s), please stop port(s) first.\n");
2837                 return;
2838         }
2839
2840         printf("Resetting ports...\n");
2841
2842         RTE_ETH_FOREACH_DEV(pi) {
2843                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2844                         continue;
2845
2846                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2847                         printf("Please remove port %d from forwarding "
2848                                "configuration.\n", pi);
2849                         continue;
2850                 }
2851
2852                 if (port_is_bonding_slave(pi)) {
2853                         printf("Please remove port %d from bonded device.\n",
2854                                pi);
2855                         continue;
2856                 }
2857
2858                 diag = rte_eth_dev_reset(pi);
2859                 if (diag == 0) {
2860                         port = &ports[pi];
2861                         port->need_reconfig = 1;
2862                         port->need_reconfig_queues = 1;
2863                 } else {
2864                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2865                 }
2866         }
2867
2868         printf("Done\n");
2869 }
2870
2871 void
2872 attach_port(char *identifier)
2873 {
2874         portid_t pi;
2875         struct rte_dev_iterator iterator;
2876
2877         printf("Attaching a new port...\n");
2878
2879         if (identifier == NULL) {
2880                 printf("Invalid parameters are specified\n");
2881                 return;
2882         }
2883
2884         if (rte_dev_probe(identifier) < 0) {
2885                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2886                 return;
2887         }
2888
2889         /* first attach mode: event */
2890         if (setup_on_probe_event) {
2891                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2892                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2893                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2894                                         ports[pi].need_setup != 0)
2895                                 setup_attached_port(pi);
2896                 return;
2897         }
2898
2899         /* second attach mode: iterator */
2900         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2901                 /* setup ports matching the devargs used for probing */
2902                 if (port_is_forwarding(pi))
2903                         continue; /* port was already attached before */
2904                 setup_attached_port(pi);
2905         }
2906 }
2907
2908 static void
2909 setup_attached_port(portid_t pi)
2910 {
2911         unsigned int socket_id;
2912         int ret;
2913
2914         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2915         /* if socket_id is invalid, set to the first available socket. */
2916         if (check_socket_id(socket_id) < 0)
2917                 socket_id = socket_ids[0];
2918         reconfig(pi, socket_id);
2919         ret = rte_eth_promiscuous_enable(pi);
2920         if (ret != 0)
2921                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2922                         pi, rte_strerror(-ret));
2923
2924         ports_ids[nb_ports++] = pi;
2925         fwd_ports_ids[nb_fwd_ports++] = pi;
2926         nb_cfg_ports = nb_fwd_ports;
2927         ports[pi].need_setup = 0;
2928         ports[pi].port_status = RTE_PORT_STOPPED;
2929
2930         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2931         printf("Done\n");
2932 }
2933
2934 static void
2935 detach_device(struct rte_device *dev)
2936 {
2937         portid_t sibling;
2938
2939         if (dev == NULL) {
2940                 printf("Device already removed\n");
2941                 return;
2942         }
2943
2944         printf("Removing a device...\n");
2945
2946         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2947                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2948                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2949                                 printf("Port %u not stopped\n", sibling);
2950                                 return;
2951                         }
2952                         port_flow_flush(sibling);
2953                 }
2954         }
2955
2956         if (rte_dev_remove(dev) < 0) {
2957                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2958                 return;
2959         }
2960         remove_invalid_ports();
2961
2962         printf("Device is detached\n");
2963         printf("Now total ports is %d\n", nb_ports);
2964         printf("Done\n");
2965         return;
2966 }
2967
2968 void
2969 detach_port_device(portid_t port_id)
2970 {
2971         if (port_id_is_invalid(port_id, ENABLED_WARN))
2972                 return;
2973
2974         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2975                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2976                         printf("Port not stopped\n");
2977                         return;
2978                 }
2979                 printf("Port was not closed\n");
2980         }
2981
2982         detach_device(rte_eth_devices[port_id].device);
2983 }
2984
2985 void
2986 detach_devargs(char *identifier)
2987 {
2988         struct rte_dev_iterator iterator;
2989         struct rte_devargs da;
2990         portid_t port_id;
2991
2992         printf("Removing a device...\n");
2993
2994         memset(&da, 0, sizeof(da));
2995         if (rte_devargs_parsef(&da, "%s", identifier)) {
2996                 printf("cannot parse identifier\n");
2997                 return;
2998         }
2999
3000         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3001                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3002                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3003                                 printf("Port %u not stopped\n", port_id);
3004                                 rte_eth_iterator_cleanup(&iterator);
3005                                 rte_devargs_reset(&da);
3006                                 return;
3007                         }
3008                         port_flow_flush(port_id);
3009                 }
3010         }
3011
3012         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3013                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3014                             da.name, da.bus->name);
3015                 rte_devargs_reset(&da);
3016                 return;
3017         }
3018
3019         remove_invalid_ports();
3020
3021         printf("Device %s is detached\n", identifier);
3022         printf("Now total ports is %d\n", nb_ports);
3023         printf("Done\n");
3024         rte_devargs_reset(&da);
3025 }
3026
3027 void
3028 pmd_test_exit(void)
3029 {
3030         portid_t pt_id;
3031         unsigned int i;
3032         int ret;
3033
3034         if (test_done == 0)
3035                 stop_packet_forwarding();
3036
3037 #ifndef RTE_EXEC_ENV_WINDOWS
3038         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3039                 if (mempools[i]) {
3040                         if (mp_alloc_type == MP_ALLOC_ANON)
3041                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3042                                                      NULL);
3043                 }
3044         }
3045 #endif
3046         if (ports != NULL) {
3047                 no_link_check = 1;
3048                 RTE_ETH_FOREACH_DEV(pt_id) {
3049                         printf("\nStopping port %d...\n", pt_id);
3050                         fflush(stdout);
3051                         stop_port(pt_id);
3052                 }
3053                 RTE_ETH_FOREACH_DEV(pt_id) {
3054                         printf("\nShutting down port %d...\n", pt_id);
3055                         fflush(stdout);
3056                         close_port(pt_id);
3057                 }
3058         }
3059
3060         if (hot_plug) {
3061                 ret = rte_dev_event_monitor_stop();
3062                 if (ret) {
3063                         RTE_LOG(ERR, EAL,
3064                                 "fail to stop device event monitor.");
3065                         return;
3066                 }
3067
3068                 ret = rte_dev_event_callback_unregister(NULL,
3069                         dev_event_callback, NULL);
3070                 if (ret < 0) {
3071                         RTE_LOG(ERR, EAL,
3072                                 "fail to unregister device event callback.\n");
3073                         return;
3074                 }
3075
3076                 ret = rte_dev_hotplug_handle_disable();
3077                 if (ret) {
3078                         RTE_LOG(ERR, EAL,
3079                                 "fail to disable hotplug handling.\n");
3080                         return;
3081                 }
3082         }
3083         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3084                 if (mempools[i])
3085                         rte_mempool_free(mempools[i]);
3086         }
3087
3088         printf("\nBye...\n");
3089 }
3090
3091 typedef void (*cmd_func_t)(void);
3092 struct pmd_test_command {
3093         const char *cmd_name;
3094         cmd_func_t cmd_func;
3095 };
3096
3097 /* Check the link status of all ports in up to 9s, and print them finally */
3098 static void
3099 check_all_ports_link_status(uint32_t port_mask)
3100 {
3101 #define CHECK_INTERVAL 100 /* 100ms */
3102 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3103         portid_t portid;
3104         uint8_t count, all_ports_up, print_flag = 0;
3105         struct rte_eth_link link;
3106         int ret;
3107         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3108
3109         printf("Checking link statuses...\n");
3110         fflush(stdout);
3111         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3112                 all_ports_up = 1;
3113                 RTE_ETH_FOREACH_DEV(portid) {
3114                         if ((port_mask & (1 << portid)) == 0)
3115                                 continue;
3116                         memset(&link, 0, sizeof(link));
3117                         ret = rte_eth_link_get_nowait(portid, &link);
3118                         if (ret < 0) {
3119                                 all_ports_up = 0;
3120                                 if (print_flag == 1)
3121                                         printf("Port %u link get failed: %s\n",
3122                                                 portid, rte_strerror(-ret));
3123                                 continue;
3124                         }
3125                         /* print link status if flag set */
3126                         if (print_flag == 1) {
3127                                 rte_eth_link_to_str(link_status,
3128                                         sizeof(link_status), &link);
3129                                 printf("Port %d %s\n", portid, link_status);
3130                                 continue;
3131                         }
3132                         /* clear all_ports_up flag if any link down */
3133                         if (link.link_status == ETH_LINK_DOWN) {
3134                                 all_ports_up = 0;
3135                                 break;
3136                         }
3137                 }
3138                 /* after finally printing all link status, get out */
3139                 if (print_flag == 1)
3140                         break;
3141
3142                 if (all_ports_up == 0) {
3143                         fflush(stdout);
3144                         rte_delay_ms(CHECK_INTERVAL);
3145                 }
3146
3147                 /* set the print_flag if all ports up or timeout */
3148                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3149                         print_flag = 1;
3150                 }
3151
3152                 if (lsc_interrupt)
3153                         break;
3154         }
3155 }
3156
3157 static void
3158 rmv_port_callback(void *arg)
3159 {
3160         int need_to_start = 0;
3161         int org_no_link_check = no_link_check;
3162         portid_t port_id = (intptr_t)arg;
3163         struct rte_device *dev;
3164
3165         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3166
3167         if (!test_done && port_is_forwarding(port_id)) {
3168                 need_to_start = 1;
3169                 stop_packet_forwarding();
3170         }
3171         no_link_check = 1;
3172         stop_port(port_id);
3173         no_link_check = org_no_link_check;
3174
3175         /* Save rte_device pointer before closing ethdev port */
3176         dev = rte_eth_devices[port_id].device;
3177         close_port(port_id);
3178         detach_device(dev); /* might be already removed or have more ports */
3179
3180         if (need_to_start)
3181                 start_packet_forwarding(0);
3182 }
3183
3184 /* This function is used by the interrupt thread */
3185 static int
3186 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3187                   void *ret_param)
3188 {
3189         RTE_SET_USED(param);
3190         RTE_SET_USED(ret_param);
3191
3192         if (type >= RTE_ETH_EVENT_MAX) {
3193                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3194                         port_id, __func__, type);
3195                 fflush(stderr);
3196         } else if (event_print_mask & (UINT32_C(1) << type)) {
3197                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3198                         eth_event_desc[type]);
3199                 fflush(stdout);
3200         }
3201
3202         switch (type) {
3203         case RTE_ETH_EVENT_NEW:
3204                 ports[port_id].need_setup = 1;
3205                 ports[port_id].port_status = RTE_PORT_HANDLING;
3206                 break;
3207         case RTE_ETH_EVENT_INTR_RMV:
3208                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3209                         break;
3210                 if (rte_eal_alarm_set(100000,
3211                                 rmv_port_callback, (void *)(intptr_t)port_id))
3212                         fprintf(stderr, "Could not set up deferred device removal\n");
3213                 break;
3214         case RTE_ETH_EVENT_DESTROY:
3215                 ports[port_id].port_status = RTE_PORT_CLOSED;
3216                 printf("Port %u is closed\n", port_id);
3217                 break;
3218         default:
3219                 break;
3220         }
3221         return 0;
3222 }
3223
3224 static int
3225 register_eth_event_callback(void)
3226 {
3227         int ret;
3228         enum rte_eth_event_type event;
3229
3230         for (event = RTE_ETH_EVENT_UNKNOWN;
3231                         event < RTE_ETH_EVENT_MAX; event++) {
3232                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3233                                 event,
3234                                 eth_event_callback,
3235                                 NULL);
3236                 if (ret != 0) {
3237                         TESTPMD_LOG(ERR, "Failed to register callback for "
3238                                         "%s event\n", eth_event_desc[event]);
3239                         return -1;
3240                 }
3241         }
3242
3243         return 0;
3244 }
3245
3246 /* This function is used by the interrupt thread */
3247 static void
3248 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3249                              __rte_unused void *arg)
3250 {
3251         uint16_t port_id;
3252         int ret;
3253
3254         if (type >= RTE_DEV_EVENT_MAX) {
3255                 fprintf(stderr, "%s called upon invalid event %d\n",
3256                         __func__, type);
3257                 fflush(stderr);
3258         }
3259
3260         switch (type) {
3261         case RTE_DEV_EVENT_REMOVE:
3262                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3263                         device_name);
3264                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3265                 if (ret) {
3266                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3267                                 device_name);
3268                         return;
3269                 }
3270                 /*
3271                  * Because the user's callback is invoked in eal interrupt
3272                  * callback, the interrupt callback need to be finished before
3273                  * it can be unregistered when detaching device. So finish
3274                  * callback soon and use a deferred removal to detach device
3275                  * is need. It is a workaround, once the device detaching be
3276                  * moved into the eal in the future, the deferred removal could
3277                  * be deleted.
3278                  */
3279                 if (rte_eal_alarm_set(100000,
3280                                 rmv_port_callback, (void *)(intptr_t)port_id))
3281                         RTE_LOG(ERR, EAL,
3282                                 "Could not set up deferred device removal\n");
3283                 break;
3284         case RTE_DEV_EVENT_ADD:
3285                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3286                         device_name);
3287                 /* TODO: After finish kernel driver binding,
3288                  * begin to attach port.
3289                  */
3290                 break;
3291         default:
3292                 break;
3293         }
3294 }
3295
3296 static void
3297 rxtx_port_config(struct rte_port *port)
3298 {
3299         uint16_t qid;
3300         uint64_t offloads;
3301
3302         for (qid = 0; qid < nb_rxq; qid++) {
3303                 offloads = port->rx_conf[qid].offloads;
3304                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3305                 if (offloads != 0)
3306                         port->rx_conf[qid].offloads = offloads;
3307
3308                 /* Check if any Rx parameters have been passed */
3309                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3310                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3311
3312                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3313                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3314
3315                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3316                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3317
3318                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3319                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3320
3321                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3322                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3323
3324                 port->nb_rx_desc[qid] = nb_rxd;
3325         }
3326
3327         for (qid = 0; qid < nb_txq; qid++) {
3328                 offloads = port->tx_conf[qid].offloads;
3329                 port->tx_conf[qid] = port->dev_info.default_txconf;
3330                 if (offloads != 0)
3331                         port->tx_conf[qid].offloads = offloads;
3332
3333                 /* Check if any Tx parameters have been passed */
3334                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3335                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3336
3337                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3338                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3339
3340                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3341                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3342
3343                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3344                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3345
3346                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3347                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3348
3349                 port->nb_tx_desc[qid] = nb_txd;
3350         }
3351 }
3352
3353 /*
3354  * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3355  * MTU is also aligned if JUMBO_FRAME offload is not set.
3356  *
3357  * port->dev_info should be set before calling this function.
3358  *
3359  * return 0 on success, negative on error
3360  */
3361 int
3362 update_jumbo_frame_offload(portid_t portid)
3363 {
3364         struct rte_port *port = &ports[portid];
3365         uint32_t eth_overhead;
3366         uint64_t rx_offloads;
3367         int ret;
3368         bool on;
3369
3370         /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3371         if (port->dev_info.max_mtu != UINT16_MAX &&
3372             port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3373                 eth_overhead = port->dev_info.max_rx_pktlen -
3374                                 port->dev_info.max_mtu;
3375         else
3376                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3377
3378         rx_offloads = port->dev_conf.rxmode.offloads;
3379
3380         /* Default config value is 0 to use PMD specific overhead */
3381         if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3382                 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3383
3384         if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3385                 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3386                 on = false;
3387         } else {
3388                 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3389                         printf("Frame size (%u) is not supported by port %u\n",
3390                                 port->dev_conf.rxmode.max_rx_pkt_len,
3391                                 portid);
3392                         return -1;
3393                 }
3394                 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3395                 on = true;
3396         }
3397
3398         if (rx_offloads != port->dev_conf.rxmode.offloads) {
3399                 uint16_t qid;
3400
3401                 port->dev_conf.rxmode.offloads = rx_offloads;
3402
3403                 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3404                 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3405                         if (on)
3406                                 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3407                         else
3408                                 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3409                 }
3410         }
3411
3412         /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3413          * if unset do it here
3414          */
3415         if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3416                 ret = rte_eth_dev_set_mtu(portid,
3417                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3418                 if (ret)
3419                         printf("Failed to set MTU to %u for port %u\n",
3420                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3421                                 portid);
3422         }
3423
3424         return 0;
3425 }
3426
3427 void
3428 init_port_config(void)
3429 {
3430         portid_t pid;
3431         struct rte_port *port;
3432         int ret;
3433
3434         RTE_ETH_FOREACH_DEV(pid) {
3435                 port = &ports[pid];
3436                 port->dev_conf.fdir_conf = fdir_conf;
3437
3438                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3439                 if (ret != 0)
3440                         return;
3441
3442                 if (nb_rxq > 1) {
3443                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3444                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3445                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3446                 } else {
3447                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3448                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3449                 }
3450
3451                 if (port->dcb_flag == 0) {
3452                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3453                                 port->dev_conf.rxmode.mq_mode =
3454                                         (enum rte_eth_rx_mq_mode)
3455                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3456                         else
3457                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3458                 }
3459
3460                 rxtx_port_config(port);
3461
3462                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3463                 if (ret != 0)
3464                         return;
3465
3466 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3467                 rte_pmd_ixgbe_bypass_init(pid);
3468 #endif
3469
3470                 if (lsc_interrupt &&
3471                     (rte_eth_devices[pid].data->dev_flags &
3472                      RTE_ETH_DEV_INTR_LSC))
3473                         port->dev_conf.intr_conf.lsc = 1;
3474                 if (rmv_interrupt &&
3475                     (rte_eth_devices[pid].data->dev_flags &
3476                      RTE_ETH_DEV_INTR_RMV))
3477                         port->dev_conf.intr_conf.rmv = 1;
3478         }
3479 }
3480
3481 void set_port_slave_flag(portid_t slave_pid)
3482 {
3483         struct rte_port *port;
3484
3485         port = &ports[slave_pid];
3486         port->slave_flag = 1;
3487 }
3488
3489 void clear_port_slave_flag(portid_t slave_pid)
3490 {
3491         struct rte_port *port;
3492
3493         port = &ports[slave_pid];
3494         port->slave_flag = 0;
3495 }
3496
3497 uint8_t port_is_bonding_slave(portid_t slave_pid)
3498 {
3499         struct rte_port *port;
3500
3501         port = &ports[slave_pid];
3502         if ((rte_eth_devices[slave_pid].data->dev_flags &
3503             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3504                 return 1;
3505         return 0;
3506 }
3507
3508 const uint16_t vlan_tags[] = {
3509                 0,  1,  2,  3,  4,  5,  6,  7,
3510                 8,  9, 10, 11,  12, 13, 14, 15,
3511                 16, 17, 18, 19, 20, 21, 22, 23,
3512                 24, 25, 26, 27, 28, 29, 30, 31
3513 };
3514
3515 static  int
3516 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3517                  enum dcb_mode_enable dcb_mode,
3518                  enum rte_eth_nb_tcs num_tcs,
3519                  uint8_t pfc_en)
3520 {
3521         uint8_t i;
3522         int32_t rc;
3523         struct rte_eth_rss_conf rss_conf;
3524
3525         /*
3526          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3527          * given above, and the number of traffic classes available for use.
3528          */
3529         if (dcb_mode == DCB_VT_ENABLED) {
3530                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3531                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3532                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3533                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3534
3535                 /* VMDQ+DCB RX and TX configurations */
3536                 vmdq_rx_conf->enable_default_pool = 0;
3537                 vmdq_rx_conf->default_pool = 0;
3538                 vmdq_rx_conf->nb_queue_pools =
3539                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3540                 vmdq_tx_conf->nb_queue_pools =
3541                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3542
3543                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3544                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3545                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3546                         vmdq_rx_conf->pool_map[i].pools =
3547                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3548                 }
3549                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3550                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3551                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3552                 }
3553
3554                 /* set DCB mode of RX and TX of multiple queues */
3555                 eth_conf->rxmode.mq_mode =
3556                                 (enum rte_eth_rx_mq_mode)
3557                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3558                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3559         } else {
3560                 struct rte_eth_dcb_rx_conf *rx_conf =
3561                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3562                 struct rte_eth_dcb_tx_conf *tx_conf =
3563                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3564
3565                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3566
3567                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3568                 if (rc != 0)
3569                         return rc;
3570
3571                 rx_conf->nb_tcs = num_tcs;
3572                 tx_conf->nb_tcs = num_tcs;
3573
3574                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3575                         rx_conf->dcb_tc[i] = i % num_tcs;
3576                         tx_conf->dcb_tc[i] = i % num_tcs;
3577                 }
3578
3579                 eth_conf->rxmode.mq_mode =
3580                                 (enum rte_eth_rx_mq_mode)
3581                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3582                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3583                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3584         }
3585
3586         if (pfc_en)
3587                 eth_conf->dcb_capability_en =
3588                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3589         else
3590                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3591
3592         return 0;
3593 }
3594
3595 int
3596 init_port_dcb_config(portid_t pid,
3597                      enum dcb_mode_enable dcb_mode,
3598                      enum rte_eth_nb_tcs num_tcs,
3599                      uint8_t pfc_en)
3600 {
3601         struct rte_eth_conf port_conf;
3602         struct rte_port *rte_port;
3603         int retval;
3604         uint16_t i;
3605
3606         rte_port = &ports[pid];
3607
3608         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3609
3610         port_conf.rxmode = rte_port->dev_conf.rxmode;
3611         port_conf.txmode = rte_port->dev_conf.txmode;
3612
3613         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3614         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3615         if (retval < 0)
3616                 return retval;
3617         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3618
3619         /* re-configure the device . */
3620         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3621         if (retval < 0)
3622                 return retval;
3623
3624         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3625         if (retval != 0)
3626                 return retval;
3627
3628         /* If dev_info.vmdq_pool_base is greater than 0,
3629          * the queue id of vmdq pools is started after pf queues.
3630          */
3631         if (dcb_mode == DCB_VT_ENABLED &&
3632             rte_port->dev_info.vmdq_pool_base > 0) {
3633                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3634                         " for port %d.", pid);
3635                 return -1;
3636         }
3637
3638         /* Assume the ports in testpmd have the same dcb capability
3639          * and has the same number of rxq and txq in dcb mode
3640          */
3641         if (dcb_mode == DCB_VT_ENABLED) {
3642                 if (rte_port->dev_info.max_vfs > 0) {
3643                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3644                         nb_txq = rte_port->dev_info.nb_tx_queues;
3645                 } else {
3646                         nb_rxq = rte_port->dev_info.max_rx_queues;
3647                         nb_txq = rte_port->dev_info.max_tx_queues;
3648                 }
3649         } else {
3650                 /*if vt is disabled, use all pf queues */
3651                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3652                         nb_rxq = rte_port->dev_info.max_rx_queues;
3653                         nb_txq = rte_port->dev_info.max_tx_queues;
3654                 } else {
3655                         nb_rxq = (queueid_t)num_tcs;
3656                         nb_txq = (queueid_t)num_tcs;
3657
3658                 }
3659         }
3660         rx_free_thresh = 64;
3661
3662         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3663
3664         rxtx_port_config(rte_port);
3665         /* VLAN filter */
3666         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3667         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3668                 rx_vft_set(pid, vlan_tags[i], 1);
3669
3670         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3671         if (retval != 0)
3672                 return retval;
3673
3674         rte_port->dcb_flag = 1;
3675
3676         /* Enter DCB configuration status */
3677         dcb_config = 1;
3678
3679         return 0;
3680 }
3681
3682 static void
3683 init_port(void)
3684 {
3685         int i;
3686
3687         /* Configuration of Ethernet ports. */
3688         ports = rte_zmalloc("testpmd: ports",
3689                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3690                             RTE_CACHE_LINE_SIZE);
3691         if (ports == NULL) {
3692                 rte_exit(EXIT_FAILURE,
3693                                 "rte_zmalloc(%d struct rte_port) failed\n",
3694                                 RTE_MAX_ETHPORTS);
3695         }
3696         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3697                 LIST_INIT(&ports[i].flow_tunnel_list);
3698         /* Initialize ports NUMA structures */
3699         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3700         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3701         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3702 }
3703
3704 static void
3705 force_quit(void)
3706 {
3707         pmd_test_exit();
3708         prompt_exit();
3709 }
3710
3711 static void
3712 print_stats(void)
3713 {
3714         uint8_t i;
3715         const char clr[] = { 27, '[', '2', 'J', '\0' };
3716         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3717
3718         /* Clear screen and move to top left */
3719         printf("%s%s", clr, top_left);
3720
3721         printf("\nPort statistics ====================================");
3722         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3723                 nic_stats_display(fwd_ports_ids[i]);
3724
3725         fflush(stdout);
3726 }
3727
3728 static void
3729 signal_handler(int signum)
3730 {
3731         if (signum == SIGINT || signum == SIGTERM) {
3732                 printf("\nSignal %d received, preparing to exit...\n",
3733                                 signum);
3734 #ifdef RTE_LIB_PDUMP
3735                 /* uninitialize packet capture framework */
3736                 rte_pdump_uninit();
3737 #endif
3738 #ifdef RTE_LIB_LATENCYSTATS
3739                 if (latencystats_enabled != 0)
3740                         rte_latencystats_uninit();
3741 #endif
3742                 force_quit();
3743                 /* Set flag to indicate the force termination. */
3744                 f_quit = 1;
3745                 /* exit with the expected status */
3746 #ifndef RTE_EXEC_ENV_WINDOWS
3747                 signal(signum, SIG_DFL);
3748                 kill(getpid(), signum);
3749 #endif
3750         }
3751 }
3752
3753 int
3754 main(int argc, char** argv)
3755 {
3756         int diag;
3757         portid_t port_id;
3758         uint16_t count;
3759         int ret;
3760
3761         signal(SIGINT, signal_handler);
3762         signal(SIGTERM, signal_handler);
3763
3764         testpmd_logtype = rte_log_register("testpmd");
3765         if (testpmd_logtype < 0)
3766                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3767         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3768
3769         diag = rte_eal_init(argc, argv);
3770         if (diag < 0)
3771                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3772                          rte_strerror(rte_errno));
3773
3774         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3775                 rte_exit(EXIT_FAILURE,
3776                          "Secondary process type not supported.\n");
3777
3778         ret = register_eth_event_callback();
3779         if (ret != 0)
3780                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3781
3782 #ifdef RTE_LIB_PDUMP
3783         /* initialize packet capture framework */
3784         rte_pdump_init();
3785 #endif
3786
3787         count = 0;
3788         RTE_ETH_FOREACH_DEV(port_id) {
3789                 ports_ids[count] = port_id;
3790                 count++;
3791         }
3792         nb_ports = (portid_t) count;
3793         if (nb_ports == 0)
3794                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3795
3796         /* allocate port structures, and init them */
3797         init_port();
3798
3799         set_def_fwd_config();
3800         if (nb_lcores == 0)
3801                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3802                          "Check the core mask argument\n");
3803
3804         /* Bitrate/latency stats disabled by default */
3805 #ifdef RTE_LIB_BITRATESTATS
3806         bitrate_enabled = 0;
3807 #endif
3808 #ifdef RTE_LIB_LATENCYSTATS
3809         latencystats_enabled = 0;
3810 #endif
3811
3812         /* on FreeBSD, mlockall() is disabled by default */
3813 #ifdef RTE_EXEC_ENV_FREEBSD
3814         do_mlockall = 0;
3815 #else
3816         do_mlockall = 1;
3817 #endif
3818
3819         argc -= diag;
3820         argv += diag;
3821         if (argc > 1)
3822                 launch_args_parse(argc, argv);
3823
3824 #ifndef RTE_EXEC_ENV_WINDOWS
3825         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3826                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3827                         strerror(errno));
3828         }
3829 #endif
3830
3831         if (tx_first && interactive)
3832                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3833                                 "interactive mode.\n");
3834
3835         if (tx_first && lsc_interrupt) {
3836                 printf("Warning: lsc_interrupt needs to be off when "
3837                                 " using tx_first. Disabling.\n");
3838                 lsc_interrupt = 0;
3839         }
3840
3841         if (!nb_rxq && !nb_txq)
3842                 printf("Warning: Either rx or tx queues should be non-zero\n");
3843
3844         if (nb_rxq > 1 && nb_rxq > nb_txq)
3845                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3846                        "but nb_txq=%d will prevent to fully test it.\n",
3847                        nb_rxq, nb_txq);
3848
3849         init_config();
3850
3851         if (hot_plug) {
3852                 ret = rte_dev_hotplug_handle_enable();
3853                 if (ret) {
3854                         RTE_LOG(ERR, EAL,
3855                                 "fail to enable hotplug handling.");
3856                         return -1;
3857                 }
3858
3859                 ret = rte_dev_event_monitor_start();
3860                 if (ret) {
3861                         RTE_LOG(ERR, EAL,
3862                                 "fail to start device event monitoring.");
3863                         return -1;
3864                 }
3865
3866                 ret = rte_dev_event_callback_register(NULL,
3867                         dev_event_callback, NULL);
3868                 if (ret) {
3869                         RTE_LOG(ERR, EAL,
3870                                 "fail  to register device event callback\n");
3871                         return -1;
3872                 }
3873         }
3874
3875         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3876                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3877
3878         /* set all ports to promiscuous mode by default */
3879         RTE_ETH_FOREACH_DEV(port_id) {
3880                 ret = rte_eth_promiscuous_enable(port_id);
3881                 if (ret != 0)
3882                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3883                                 port_id, rte_strerror(-ret));
3884         }
3885
3886         /* Init metrics library */
3887         rte_metrics_init(rte_socket_id());
3888
3889 #ifdef RTE_LIB_LATENCYSTATS
3890         if (latencystats_enabled != 0) {
3891                 int ret = rte_latencystats_init(1, NULL);
3892                 if (ret)
3893                         printf("Warning: latencystats init()"
3894                                 " returned error %d\n", ret);
3895                 printf("Latencystats running on lcore %d\n",
3896                         latencystats_lcore_id);
3897         }
3898 #endif
3899
3900         /* Setup bitrate stats */
3901 #ifdef RTE_LIB_BITRATESTATS
3902         if (bitrate_enabled != 0) {
3903                 bitrate_data = rte_stats_bitrate_create();
3904                 if (bitrate_data == NULL)
3905                         rte_exit(EXIT_FAILURE,
3906                                 "Could not allocate bitrate data.\n");
3907                 rte_stats_bitrate_reg(bitrate_data);
3908         }
3909 #endif
3910
3911 #ifdef RTE_LIB_CMDLINE
3912         if (strlen(cmdline_filename) != 0)
3913                 cmdline_read_from_file(cmdline_filename);
3914
3915         if (interactive == 1) {
3916                 if (auto_start) {
3917                         printf("Start automatic packet forwarding\n");
3918                         start_packet_forwarding(0);
3919                 }
3920                 prompt();
3921                 pmd_test_exit();
3922         } else
3923 #endif
3924         {
3925                 char c;
3926                 int rc;
3927
3928                 f_quit = 0;
3929
3930                 printf("No commandline core given, start packet forwarding\n");
3931                 start_packet_forwarding(tx_first);
3932                 if (stats_period != 0) {
3933                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3934                         uint64_t timer_period;
3935
3936                         /* Convert to number of cycles */
3937                         timer_period = stats_period * rte_get_timer_hz();
3938
3939                         while (f_quit == 0) {
3940                                 cur_time = rte_get_timer_cycles();
3941                                 diff_time += cur_time - prev_time;
3942
3943                                 if (diff_time >= timer_period) {
3944                                         print_stats();
3945                                         /* Reset the timer */
3946                                         diff_time = 0;
3947                                 }
3948                                 /* Sleep to avoid unnecessary checks */
3949                                 prev_time = cur_time;
3950                                 rte_delay_us_sleep(US_PER_S);
3951                         }
3952                 }
3953
3954                 printf("Press enter to exit\n");
3955                 rc = read(0, &c, 1);
3956                 pmd_test_exit();
3957                 if (rc < 0)
3958                         return 1;
3959         }
3960
3961         ret = rte_eal_cleanup();
3962         if (ret != 0)
3963                 rte_exit(EXIT_FAILURE,
3964                          "EAL cleanup failed: %s\n", strerror(-ret));
3965
3966         return EXIT_SUCCESS;
3967 }