app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATESTATS
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 200         DEFAULT_MBUF_DATA_SIZE
 201 }; /**< Mbuf data space size. */
 202 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 203                                       * specified on command-line. */
 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 205
 206 /*
 207  * In container, it cannot terminate the process which running with 'stats-period'
 208  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 209  */
 210 uint8_t f_quit;
 211
 212 /*
 213  * Configuration of packet segments used to scatter received packets
 214  * if some of split features is configured.
 215  */
 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 217 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 219 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 220
 221 /*
 222  * Configuration of packet segments used by the "txonly" processing engine.
 223  */
 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 226         TXONLY_DEF_PACKET_LEN,
 227 };
 228 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 229
 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 231 /**< Split policy for packets to TX. */
 232
 233 uint8_t txonly_multi_flow;
 234 /**< Whether multiple flows are generated in TXONLY mode. */
 235
 236 uint32_t tx_pkt_times_inter;
 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 238
 239 uint32_t tx_pkt_times_intra;
 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 241
 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 243 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 244
 245 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 246 uint8_t dcb_config = 0;
 247
 248 /* Whether the dcb is in testing status */
 249 uint8_t dcb_test = 0;
 250
 251 /*
 252  * Configurable number of RX/TX queues.
 253  */
 254 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 255 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 256 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 257
 258 /*
 259  * Configurable number of RX/TX ring descriptors.
 260  * Defaults are supplied by drivers via ethdev.
 261  */
 262 #define RTE_TEST_RX_DESC_DEFAULT 0
 263 #define RTE_TEST_TX_DESC_DEFAULT 0
 264 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 265 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 266
 267 #define RTE_PMD_PARAM_UNSET -1
 268 /*
 269  * Configurable values of RX and TX ring threshold registers.
 270  */
 271
 272 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 273 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 274 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 275
 276 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 277 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 278 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 279
 280 /*
 281  * Configurable value of RX free threshold.
 282  */
 283 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 284
 285 /*
 286  * Configurable value of RX drop enable.
 287  */
 288 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 289
 290 /*
 291  * Configurable value of TX free threshold.
 292  */
 293 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 294
 295 /*
 296  * Configurable value of TX RS bit threshold.
 297  */
 298 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 299
 300 /*
 301  * Configurable value of buffered packets before sending.
 302  */
 303 uint16_t noisy_tx_sw_bufsz;
 304
 305 /*
 306  * Configurable value of packet buffer timeout.
 307  */
 308 uint16_t noisy_tx_sw_buf_flush_time;
 309
 310 /*
 311  * Configurable value for size of VNF internal memory area
 312  * used for simulating noisy neighbour behaviour
 313  */
 314 uint64_t noisy_lkup_mem_sz;
 315
 316 /*
 317  * Configurable value of number of random writes done in
 318  * VNF simulation memory area.
 319  */
 320 uint64_t noisy_lkup_num_writes;
 321
 322 /*
 323  * Configurable value of number of random reads done in
 324  * VNF simulation memory area.
 325  */
 326 uint64_t noisy_lkup_num_reads;
 327
 328 /*
 329  * Configurable value of number of random reads/writes done in
 330  * VNF simulation memory area.
 331  */
 332 uint64_t noisy_lkup_num_reads_writes;
 333
 334 /*
 335  * Receive Side Scaling (RSS) configuration.
 336  */
 337 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 338
 339 /*
 340  * Port topology configuration
 341  */
 342 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 343
 344 /*
 345  * Avoids to flush all the RX streams before starts forwarding.
 346  */
 347 uint8_t no_flush_rx = 0; /* flush by default */
 348
 349 /*
 350  * Flow API isolated mode.
 351  */
 352 uint8_t flow_isolate_all;
 353
 354 /*
 355  * Avoids to check link status when starting/stopping a port.
 356  */
 357 uint8_t no_link_check = 0; /* check by default */
 358
 359 /*
 360  * Don't automatically start all ports in interactive mode.
 361  */
 362 uint8_t no_device_start = 0;
 363
 364 /*
 365  * Enable link status change notification
 366  */
 367 uint8_t lsc_interrupt = 1; /* enabled by default */
 368
 369 /*
 370  * Enable device removal notification.
 371  */
 372 uint8_t rmv_interrupt = 1; /* enabled by default */
 373
 374 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 375
 376 /* After attach, port setup is called on event or by iterator */
 377 bool setup_on_probe_event = true;
 378
 379 /* Clear ptypes on port initialization. */
 380 uint8_t clear_ptypes = true;
 381
 382 /* Hairpin ports configuration mode. */
 383 uint16_t hairpin_mode;
 384
 385 /* Pretty printing of ethdev events */
 386 static const char * const eth_event_desc[] = {
 387         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 388         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 389         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 390         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 391         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 392         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 393         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 394         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 395         [RTE_ETH_EVENT_NEW] = "device probed",
 396         [RTE_ETH_EVENT_DESTROY] = "device released",
 397         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 398         [RTE_ETH_EVENT_MAX] = NULL,
 399 };
 400
 401 /*
 402  * Display or mask ether events
 403  * Default to all events except VF_MBOX
 404  */
 405 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 406                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 407                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 408                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 413 /*
 414  * Decide if all memory are locked for performance.
 415  */
 416 int do_mlockall = 0;
 417
 418 /*
 419  * NIC bypass mode configuration options.
 420  */
 421
 422 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 423 /* The NIC bypass watchdog timeout. */
 424 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 425 #endif
 426
 427
 428 #ifdef RTE_LIBRTE_LATENCY_STATS
 429
 430 /*
 431  * Set when latency stats is enabled in the commandline
 432  */
 433 uint8_t latencystats_enabled;
 434
 435 /*
 436  * Lcore ID to serive latency statistics.
 437  */
 438 lcoreid_t latencystats_lcore_id = -1;
 439
 440 #endif
 441
 442 /*
 443  * Ethernet device configuration.
 444  */
 445 struct rte_eth_rxmode rx_mode = {
 446         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 447                 /**< Default maximum frame length. */
 448 };
 449
 450 struct rte_eth_txmode tx_mode = {
 451         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 452 };
 453
 454 struct rte_fdir_conf fdir_conf = {
 455         .mode = RTE_FDIR_MODE_NONE,
 456         .pballoc = RTE_FDIR_PBALLOC_64K,
 457         .status = RTE_FDIR_REPORT_STATUS,
 458         .mask = {
 459                 .vlan_tci_mask = 0xFFEF,
 460                 .ipv4_mask     = {
 461                         .src_ip = 0xFFFFFFFF,
 462                         .dst_ip = 0xFFFFFFFF,
 463                 },
 464                 .ipv6_mask     = {
 465                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 466                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 467                 },
 468                 .src_port_mask = 0xFFFF,
 469                 .dst_port_mask = 0xFFFF,
 470                 .mac_addr_byte_mask = 0xFF,
 471                 .tunnel_type_mask = 1,
 472                 .tunnel_id_mask = 0xFFFFFFFF,
 473         },
 474         .drop_queue = 127,
 475 };
 476
 477 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 478
 479 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 480 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 481
 482 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 483 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 484
 485 uint16_t nb_tx_queue_stats_mappings = 0;
 486 uint16_t nb_rx_queue_stats_mappings = 0;
 487
 488 /*
 489  * Display zero values by default for xstats
 490  */
 491 uint8_t xstats_hide_zero;
 492
 493 /*
 494  * Measure of CPU cycles disabled by default
 495  */
 496 uint8_t record_core_cycles;
 497
 498 /*
 499  * Display of RX and TX bursts disabled by default
 500  */
 501 uint8_t record_burst_stats;
 502
 503 unsigned int num_sockets = 0;
 504 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 505
 506 #ifdef RTE_LIBRTE_BITRATESTATS
 507 /* Bitrate statistics */
 508 struct rte_stats_bitrates *bitrate_data;
 509 lcoreid_t bitrate_lcore_id;
 510 uint8_t bitrate_enabled;
 511 #endif
 512
 513 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 514 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 515
 516 /*
 517  * hexadecimal bitmask of RX mq mode can be enabled.
 518  */
 519 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 520
 521 /* Forward function declarations */
 522 static void setup_attached_port(portid_t pi);
 523 static void map_port_queue_stats_mapping_registers(portid_t pi,
 524                                                    struct rte_port *port);
 525 static void check_all_ports_link_status(uint32_t port_mask);
 526 static int eth_event_callback(portid_t port_id,
 527                               enum rte_eth_event_type type,
 528                               void *param, void *ret_param);
 529 static void dev_event_callback(const char *device_name,
 530                                 enum rte_dev_event_type type,
 531                                 void *param);
 532
 533 /*
 534  * Check if all the ports are started.
 535  * If yes, return positive value. If not, return zero.
 536  */
 537 static int all_ports_started(void);
 538
 539 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 540 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 541
 542 /* Holds the registered mbuf dynamic flags names. */
 543 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 544
 545 /*
 546  * Helper function to check if socket is already discovered.
 547  * If yes, return positive value. If not, return zero.
 548  */
 549 int
 550 new_socket_id(unsigned int socket_id)
 551 {
 552         unsigned int i;
 553
 554         for (i = 0; i < num_sockets; i++) {
 555                 if (socket_ids[i] == socket_id)
 556                         return 0;
 557         }
 558         return 1;
 559 }
 560
 561 /*
 562  * Setup default configuration.
 563  */
 564 static void
 565 set_default_fwd_lcores_config(void)
 566 {
 567         unsigned int i;
 568         unsigned int nb_lc;
 569         unsigned int sock_num;
 570
 571         nb_lc = 0;
 572         for (i = 0; i < RTE_MAX_LCORE; i++) {
 573                 if (!rte_lcore_is_enabled(i))
 574                         continue;
 575                 sock_num = rte_lcore_to_socket_id(i);
 576                 if (new_socket_id(sock_num)) {
 577                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 578                                 rte_exit(EXIT_FAILURE,
 579                                          "Total sockets greater than %u\n",
 580                                          RTE_MAX_NUMA_NODES);
 581                         }
 582                         socket_ids[num_sockets++] = sock_num;
 583                 }
 584                 if (i == rte_get_master_lcore())
 585                         continue;
 586                 fwd_lcores_cpuids[nb_lc++] = i;
 587         }
 588         nb_lcores = (lcoreid_t) nb_lc;
 589         nb_cfg_lcores = nb_lcores;
 590         nb_fwd_lcores = 1;
 591 }
 592
 593 static void
 594 set_def_peer_eth_addrs(void)
 595 {
 596         portid_t i;
 597
 598         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 599                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 600                 peer_eth_addrs[i].addr_bytes[5] = i;
 601         }
 602 }
 603
 604 static void
 605 set_default_fwd_ports_config(void)
 606 {
 607         portid_t pt_id;
 608         int i = 0;
 609
 610         RTE_ETH_FOREACH_DEV(pt_id) {
 611                 fwd_ports_ids[i++] = pt_id;
 612
 613                 /* Update sockets info according to the attached device */
 614                 int socket_id = rte_eth_dev_socket_id(pt_id);
 615                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 616                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 617                                 rte_exit(EXIT_FAILURE,
 618                                          "Total sockets greater than %u\n",
 619                                          RTE_MAX_NUMA_NODES);
 620                         }
 621                         socket_ids[num_sockets++] = socket_id;
 622                 }
 623         }
 624
 625         nb_cfg_ports = nb_ports;
 626         nb_fwd_ports = nb_ports;
 627 }
 628
 629 void
 630 set_def_fwd_config(void)
 631 {
 632         set_default_fwd_lcores_config();
 633         set_def_peer_eth_addrs();
 634         set_default_fwd_ports_config();
 635 }
 636
 637 /* extremely pessimistic estimation of memory required to create a mempool */
 638 static int
 639 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 640 {
 641         unsigned int n_pages, mbuf_per_pg, leftover;
 642         uint64_t total_mem, mbuf_mem, obj_sz;
 643
 644         /* there is no good way to predict how much space the mempool will
 645          * occupy because it will allocate chunks on the fly, and some of those
 646          * will come from default DPDK memory while some will come from our
 647          * external memory, so just assume 128MB will be enough for everyone.
 648          */
 649         uint64_t hdr_mem = 128 << 20;
 650
 651         /* account for possible non-contiguousness */
 652         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 653         if (obj_sz > pgsz) {
 654                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 655                 return -1;
 656         }
 657
 658         mbuf_per_pg = pgsz / obj_sz;
 659         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 660         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 661
 662         mbuf_mem = n_pages * pgsz;
 663
 664         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 665
 666         if (total_mem > SIZE_MAX) {
 667                 TESTPMD_LOG(ERR, "Memory size too big\n");
 668                 return -1;
 669         }
 670         *out = (size_t)total_mem;
 671
 672         return 0;
 673 }
 674
 675 static int
 676 pagesz_flags(uint64_t page_sz)
 677 {
 678         /* as per mmap() manpage, all page sizes are log2 of page size
 679          * shifted by MAP_HUGE_SHIFT
 680          */
 681         int log2 = rte_log2_u64(page_sz);
 682
 683         return (log2 << HUGE_SHIFT);
 684 }
 685
 686 static void *
 687 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 688 {
 689         void *addr;
 690         int flags;
 691
 692         /* allocate anonymous hugepages */
 693         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 694         if (huge)
 695                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 696
 697         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 698         if (addr == MAP_FAILED)
 699                 return NULL;
 700
 701         return addr;
 702 }
 703
 704 struct extmem_param {
 705         void *addr;
 706         size_t len;
 707         size_t pgsz;
 708         rte_iova_t *iova_table;
 709         unsigned int iova_table_len;
 710 };
 711
 712 static int
 713 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 714                 bool huge)
 715 {
 716         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 717                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 718         unsigned int cur_page, n_pages, pgsz_idx;
 719         size_t mem_sz, cur_pgsz;
 720         rte_iova_t *iovas = NULL;
 721         void *addr;
 722         int ret;
 723
 724         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 725                 /* skip anything that is too big */
 726                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 727                         continue;
 728
 729                 cur_pgsz = pgsizes[pgsz_idx];
 730
 731                 /* if we were told not to allocate hugepages, override */
 732                 if (!huge)
 733                         cur_pgsz = sysconf(_SC_PAGESIZE);
 734
 735                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 736                 if (ret < 0) {
 737                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 738                         return -1;
 739                 }
 740
 741                 /* allocate our memory */
 742                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 743
 744                 /* if we couldn't allocate memory with a specified page size,
 745                  * that doesn't mean we can't do it with other page sizes, so
 746                  * try another one.
 747                  */
 748                 if (addr == NULL)
 749                         continue;
 750
 751                 /* store IOVA addresses for every page in this memory area */
 752                 n_pages = mem_sz / cur_pgsz;
 753
 754                 iovas = malloc(sizeof(*iovas) * n_pages);
 755
 756                 if (iovas == NULL) {
 757                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 758                         goto fail;
 759                 }
 760                 /* lock memory if it's not huge pages */
 761                 if (!huge)
 762                         mlock(addr, mem_sz);
 763
 764                 /* populate IOVA addresses */
 765                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 766                         rte_iova_t iova;
 767                         size_t offset;
 768                         void *cur;
 769
 770                         offset = cur_pgsz * cur_page;
 771                         cur = RTE_PTR_ADD(addr, offset);
 772
 773                         /* touch the page before getting its IOVA */
 774                         *(volatile char *)cur = 0;
 775
 776                         iova = rte_mem_virt2iova(cur);
 777
 778                         iovas[cur_page] = iova;
 779                 }
 780
 781                 break;
 782         }
 783         /* if we couldn't allocate anything */
 784         if (iovas == NULL)
 785                 return -1;
 786
 787         param->addr = addr;
 788         param->len = mem_sz;
 789         param->pgsz = cur_pgsz;
 790         param->iova_table = iovas;
 791         param->iova_table_len = n_pages;
 792
 793         return 0;
 794 fail:
 795         if (iovas)
 796                 free(iovas);
 797         if (addr)
 798                 munmap(addr, mem_sz);
 799
 800         return -1;
 801 }
 802
 803 static int
 804 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 805 {
 806         struct extmem_param param;
 807         int socket_id, ret;
 808
 809         memset(&param, 0, sizeof(param));
 810
 811         /* check if our heap exists */
 812         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 813         if (socket_id < 0) {
 814                 /* create our heap */
 815                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 816                 if (ret < 0) {
 817                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 818                         return -1;
 819                 }
 820         }
 821
 822         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 823         if (ret < 0) {
 824                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 825                 return -1;
 826         }
 827
 828         /* we now have a valid memory area, so add it to heap */
 829         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 830                         param.addr, param.len, param.iova_table,
 831                         param.iova_table_len, param.pgsz);
 832
 833         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 834
 835         /* not needed any more */
 836         free(param.iova_table);
 837
 838         if (ret < 0) {
 839                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 840                 munmap(param.addr, param.len);
 841                 return -1;
 842         }
 843
 844         /* success */
 845
 846         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 847                         param.len >> 20);
 848
 849         return 0;
 850 }
 851 static void
 852 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 853              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 854 {
 855         uint16_t pid = 0;
 856         int ret;
 857
 858         RTE_ETH_FOREACH_DEV(pid) {
 859                 struct rte_eth_dev *dev =
 860                         &rte_eth_devices[pid];
 861
 862                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 863                                         memhdr->len);
 864                 if (ret) {
 865                         TESTPMD_LOG(DEBUG,
 866                                     "unable to DMA unmap addr 0x%p "
 867                                     "for device %s\n",
 868                                     memhdr->addr, dev->data->name);
 869                 }
 870         }
 871         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 872         if (ret) {
 873                 TESTPMD_LOG(DEBUG,
 874                             "unable to un-register addr 0x%p\n", memhdr->addr);
 875         }
 876 }
 877
 878 static void
 879 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 880            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 881 {
 882         uint16_t pid = 0;
 883         size_t page_size = sysconf(_SC_PAGESIZE);
 884         int ret;
 885
 886         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 887                                   page_size);
 888         if (ret) {
 889                 TESTPMD_LOG(DEBUG,
 890                             "unable to register addr 0x%p\n", memhdr->addr);
 891                 return;
 892         }
 893         RTE_ETH_FOREACH_DEV(pid) {
 894                 struct rte_eth_dev *dev =
 895                         &rte_eth_devices[pid];
 896
 897                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 898                                       memhdr->len);
 899                 if (ret) {
 900                         TESTPMD_LOG(DEBUG,
 901                                     "unable to DMA map addr 0x%p "
 902                                     "for device %s\n",
 903                                     memhdr->addr, dev->data->name);
 904                 }
 905         }
 906 }
 907
 908 static unsigned int
 909 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 910             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 911 {
 912         struct rte_pktmbuf_extmem *xmem;
 913         unsigned int ext_num, zone_num, elt_num;
 914         uint16_t elt_size;
 915
 916         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 917         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 918         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 919
 920         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 921         if (xmem == NULL) {
 922                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 923                                  "external buffer descriptors\n");
 924                 *ext_mem = NULL;
 925                 return 0;
 926         }
 927         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 928                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 929                 const struct rte_memzone *mz;
 930                 char mz_name[RTE_MEMZONE_NAMESIZE];
 931                 int ret;
 932
 933                 ret = snprintf(mz_name, sizeof(mz_name),
 934                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 935                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 936                         errno = ENAMETOOLONG;
 937                         ext_num = 0;
 938                         break;
 939                 }
 940                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 941                                                  socket_id,
 942                                                  RTE_MEMZONE_IOVA_CONTIG |
 943                                                  RTE_MEMZONE_1GB |
 944                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 945                                                  EXTBUF_ZONE_SIZE);
 946                 if (mz == NULL) {
 947                         /*
 948                          * The caller exits on external buffer creation
 949                          * error, so there is no need to free memzones.
 950                          */
 951                         errno = ENOMEM;
 952                         ext_num = 0;
 953                         break;
 954                 }
 955                 xseg->buf_ptr = mz->addr;
 956                 xseg->buf_iova = mz->iova;
 957                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 958                 xseg->elt_size = elt_size;
 959         }
 960         if (ext_num == 0 && xmem != NULL) {
 961                 free(xmem);
 962                 xmem = NULL;
 963         }
 964         *ext_mem = xmem;
 965         return ext_num;
 966 }
 967
 968 /*
 969  * Configuration initialisation done once at init time.
 970  */
 971 static struct rte_mempool *
 972 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 973                  unsigned int socket_id, uint16_t size_idx)
 974 {
 975         char pool_name[RTE_MEMPOOL_NAMESIZE];
 976         struct rte_mempool *rte_mp = NULL;
 977         uint32_t mb_size;
 978
 979         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 980         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 981
 982         TESTPMD_LOG(INFO,
 983                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 984                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 985
 986         switch (mp_alloc_type) {
 987         case MP_ALLOC_NATIVE:
 988                 {
 989                         /* wrapper to rte_mempool_create() */
 990                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 991                                         rte_mbuf_best_mempool_ops());
 992                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 993                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 994                         break;
 995                 }
 996         case MP_ALLOC_ANON:
 997                 {
 998                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 999                                 mb_size, (unsigned int) mb_mempool_cache,
1000                                 sizeof(struct rte_pktmbuf_pool_private),
1001                                 socket_id, mempool_flags);
1002                         if (rte_mp == NULL)
1003                                 goto err;
1004
1005                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1006                                 rte_mempool_free(rte_mp);
1007                                 rte_mp = NULL;
1008                                 goto err;
1009                         }
1010                         rte_pktmbuf_pool_init(rte_mp, NULL);
1011                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1012                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1013                         break;
1014                 }
1015         case MP_ALLOC_XMEM:
1016         case MP_ALLOC_XMEM_HUGE:
1017                 {
1018                         int heap_socket;
1019                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1020
1021                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1022                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1023
1024                         heap_socket =
1025                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1026                         if (heap_socket < 0)
1027                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1028
1029                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1030                                         rte_mbuf_best_mempool_ops());
1031                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1032                                         mb_mempool_cache, 0, mbuf_seg_size,
1033                                         heap_socket);
1034                         break;
1035                 }
1036         case MP_ALLOC_XBUF:
1037                 {
1038                         struct rte_pktmbuf_extmem *ext_mem;
1039                         unsigned int ext_num;
1040
1041                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1042                                                socket_id, pool_name, &ext_mem);
1043                         if (ext_num == 0)
1044                                 rte_exit(EXIT_FAILURE,
1045                                          "Can't create pinned data buffers\n");
1046
1047                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1048                                         rte_mbuf_best_mempool_ops());
1049                         rte_mp = rte_pktmbuf_pool_create_extbuf
1050                                         (pool_name, nb_mbuf, mb_mempool_cache,
1051                                          0, mbuf_seg_size, socket_id,
1052                                          ext_mem, ext_num);
1053                         free(ext_mem);
1054                         break;
1055                 }
1056         default:
1057                 {
1058                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1059                 }
1060         }
1061
1062 err:
1063         if (rte_mp == NULL) {
1064                 rte_exit(EXIT_FAILURE,
1065                         "Creation of mbuf pool for socket %u failed: %s\n",
1066                         socket_id, rte_strerror(rte_errno));
1067         } else if (verbose_level > 0) {
1068                 rte_mempool_dump(stdout, rte_mp);
1069         }
1070         return rte_mp;
1071 }
1072
1073 /*
1074  * Check given socket id is valid or not with NUMA mode,
1075  * if valid, return 0, else return -1
1076  */
1077 static int
1078 check_socket_id(const unsigned int socket_id)
1079 {
1080         static int warning_once = 0;
1081
1082         if (new_socket_id(socket_id)) {
1083                 if (!warning_once && numa_support)
1084                         printf("Warning: NUMA should be configured manually by"
1085                                " using --port-numa-config and"
1086                                " --ring-numa-config parameters along with"
1087                                " --numa.\n");
1088                 warning_once = 1;
1089                 return -1;
1090         }
1091         return 0;
1092 }
1093
1094 /*
1095  * Get the allowed maximum number of RX queues.
1096  * *pid return the port id which has minimal value of
1097  * max_rx_queues in all ports.
1098  */
1099 queueid_t
1100 get_allowed_max_nb_rxq(portid_t *pid)
1101 {
1102         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1103         bool max_rxq_valid = false;
1104         portid_t pi;
1105         struct rte_eth_dev_info dev_info;
1106
1107         RTE_ETH_FOREACH_DEV(pi) {
1108                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1109                         continue;
1110
1111                 max_rxq_valid = true;
1112                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1113                         allowed_max_rxq = dev_info.max_rx_queues;
1114                         *pid = pi;
1115                 }
1116         }
1117         return max_rxq_valid ? allowed_max_rxq : 0;
1118 }
1119
1120 /*
1121  * Check input rxq is valid or not.
1122  * If input rxq is not greater than any of maximum number
1123  * of RX queues of all ports, it is valid.
1124  * if valid, return 0, else return -1
1125  */
1126 int
1127 check_nb_rxq(queueid_t rxq)
1128 {
1129         queueid_t allowed_max_rxq;
1130         portid_t pid = 0;
1131
1132         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1133         if (rxq > allowed_max_rxq) {
1134                 printf("Fail: input rxq (%u) can't be greater "
1135                        "than max_rx_queues (%u) of port %u\n",
1136                        rxq,
1137                        allowed_max_rxq,
1138                        pid);
1139                 return -1;
1140         }
1141         return 0;
1142 }
1143
1144 /*
1145  * Get the allowed maximum number of TX queues.
1146  * *pid return the port id which has minimal value of
1147  * max_tx_queues in all ports.
1148  */
1149 queueid_t
1150 get_allowed_max_nb_txq(portid_t *pid)
1151 {
1152         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1153         bool max_txq_valid = false;
1154         portid_t pi;
1155         struct rte_eth_dev_info dev_info;
1156
1157         RTE_ETH_FOREACH_DEV(pi) {
1158                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1159                         continue;
1160
1161                 max_txq_valid = true;
1162                 if (dev_info.max_tx_queues < allowed_max_txq) {
1163                         allowed_max_txq = dev_info.max_tx_queues;
1164                         *pid = pi;
1165                 }
1166         }
1167         return max_txq_valid ? allowed_max_txq : 0;
1168 }
1169
1170 /*
1171  * Check input txq is valid or not.
1172  * If input txq is not greater than any of maximum number
1173  * of TX queues of all ports, it is valid.
1174  * if valid, return 0, else return -1
1175  */
1176 int
1177 check_nb_txq(queueid_t txq)
1178 {
1179         queueid_t allowed_max_txq;
1180         portid_t pid = 0;
1181
1182         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1183         if (txq > allowed_max_txq) {
1184                 printf("Fail: input txq (%u) can't be greater "
1185                        "than max_tx_queues (%u) of port %u\n",
1186                        txq,
1187                        allowed_max_txq,
1188                        pid);
1189                 return -1;
1190         }
1191         return 0;
1192 }
1193
1194 /*
1195  * Get the allowed maximum number of RXDs of every rx queue.
1196  * *pid return the port id which has minimal value of
1197  * max_rxd in all queues of all ports.
1198  */
1199 static uint16_t
1200 get_allowed_max_nb_rxd(portid_t *pid)
1201 {
1202         uint16_t allowed_max_rxd = UINT16_MAX;
1203         portid_t pi;
1204         struct rte_eth_dev_info dev_info;
1205
1206         RTE_ETH_FOREACH_DEV(pi) {
1207                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1208                         continue;
1209
1210                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1211                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1212                         *pid = pi;
1213                 }
1214         }
1215         return allowed_max_rxd;
1216 }
1217
1218 /*
1219  * Get the allowed minimal number of RXDs of every rx queue.
1220  * *pid return the port id which has minimal value of
1221  * min_rxd in all queues of all ports.
1222  */
1223 static uint16_t
1224 get_allowed_min_nb_rxd(portid_t *pid)
1225 {
1226         uint16_t allowed_min_rxd = 0;
1227         portid_t pi;
1228         struct rte_eth_dev_info dev_info;
1229
1230         RTE_ETH_FOREACH_DEV(pi) {
1231                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1232                         continue;
1233
1234                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1235                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1236                         *pid = pi;
1237                 }
1238         }
1239
1240         return allowed_min_rxd;
1241 }
1242
1243 /*
1244  * Check input rxd is valid or not.
1245  * If input rxd is not greater than any of maximum number
1246  * of RXDs of every Rx queues and is not less than any of
1247  * minimal number of RXDs of every Rx queues, it is valid.
1248  * if valid, return 0, else return -1
1249  */
1250 int
1251 check_nb_rxd(queueid_t rxd)
1252 {
1253         uint16_t allowed_max_rxd;
1254         uint16_t allowed_min_rxd;
1255         portid_t pid = 0;
1256
1257         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1258         if (rxd > allowed_max_rxd) {
1259                 printf("Fail: input rxd (%u) can't be greater "
1260                        "than max_rxds (%u) of port %u\n",
1261                        rxd,
1262                        allowed_max_rxd,
1263                        pid);
1264                 return -1;
1265         }
1266
1267         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1268         if (rxd < allowed_min_rxd) {
1269                 printf("Fail: input rxd (%u) can't be less "
1270                        "than min_rxds (%u) of port %u\n",
1271                        rxd,
1272                        allowed_min_rxd,
1273                        pid);
1274                 return -1;
1275         }
1276
1277         return 0;
1278 }
1279
1280 /*
1281  * Get the allowed maximum number of TXDs of every rx queues.
1282  * *pid return the port id which has minimal value of
1283  * max_txd in every tx queue.
1284  */
1285 static uint16_t
1286 get_allowed_max_nb_txd(portid_t *pid)
1287 {
1288         uint16_t allowed_max_txd = UINT16_MAX;
1289         portid_t pi;
1290         struct rte_eth_dev_info dev_info;
1291
1292         RTE_ETH_FOREACH_DEV(pi) {
1293                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1294                         continue;
1295
1296                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1297                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1298                         *pid = pi;
1299                 }
1300         }
1301         return allowed_max_txd;
1302 }
1303
1304 /*
1305  * Get the allowed maximum number of TXDs of every tx queues.
1306  * *pid return the port id which has minimal value of
1307  * min_txd in every tx queue.
1308  */
1309 static uint16_t
1310 get_allowed_min_nb_txd(portid_t *pid)
1311 {
1312         uint16_t allowed_min_txd = 0;
1313         portid_t pi;
1314         struct rte_eth_dev_info dev_info;
1315
1316         RTE_ETH_FOREACH_DEV(pi) {
1317                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1318                         continue;
1319
1320                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1321                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1322                         *pid = pi;
1323                 }
1324         }
1325
1326         return allowed_min_txd;
1327 }
1328
1329 /*
1330  * Check input txd is valid or not.
1331  * If input txd is not greater than any of maximum number
1332  * of TXDs of every Rx queues, it is valid.
1333  * if valid, return 0, else return -1
1334  */
1335 int
1336 check_nb_txd(queueid_t txd)
1337 {
1338         uint16_t allowed_max_txd;
1339         uint16_t allowed_min_txd;
1340         portid_t pid = 0;
1341
1342         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1343         if (txd > allowed_max_txd) {
1344                 printf("Fail: input txd (%u) can't be greater "
1345                        "than max_txds (%u) of port %u\n",
1346                        txd,
1347                        allowed_max_txd,
1348                        pid);
1349                 return -1;
1350         }
1351
1352         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1353         if (txd < allowed_min_txd) {
1354                 printf("Fail: input txd (%u) can't be less "
1355                        "than min_txds (%u) of port %u\n",
1356                        txd,
1357                        allowed_min_txd,
1358                        pid);
1359                 return -1;
1360         }
1361         return 0;
1362 }
1363
1364
1365 /*
1366  * Get the allowed maximum number of hairpin queues.
1367  * *pid return the port id which has minimal value of
1368  * max_hairpin_queues in all ports.
1369  */
1370 queueid_t
1371 get_allowed_max_nb_hairpinq(portid_t *pid)
1372 {
1373         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1374         portid_t pi;
1375         struct rte_eth_hairpin_cap cap;
1376
1377         RTE_ETH_FOREACH_DEV(pi) {
1378                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1379                         *pid = pi;
1380                         return 0;
1381                 }
1382                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1383                         allowed_max_hairpinq = cap.max_nb_queues;
1384                         *pid = pi;
1385                 }
1386         }
1387         return allowed_max_hairpinq;
1388 }
1389
1390 /*
1391  * Check input hairpin is valid or not.
1392  * If input hairpin is not greater than any of maximum number
1393  * of hairpin queues of all ports, it is valid.
1394  * if valid, return 0, else return -1
1395  */
1396 int
1397 check_nb_hairpinq(queueid_t hairpinq)
1398 {
1399         queueid_t allowed_max_hairpinq;
1400         portid_t pid = 0;
1401
1402         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1403         if (hairpinq > allowed_max_hairpinq) {
1404                 printf("Fail: input hairpin (%u) can't be greater "
1405                        "than max_hairpin_queues (%u) of port %u\n",
1406                        hairpinq, allowed_max_hairpinq, pid);
1407                 return -1;
1408         }
1409         return 0;
1410 }
1411
1412 static void
1413 init_config(void)
1414 {
1415         portid_t pid;
1416         struct rte_port *port;
1417         struct rte_mempool *mbp;
1418         unsigned int nb_mbuf_per_pool;
1419         lcoreid_t  lc_id;
1420         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1421         struct rte_gro_param gro_param;
1422         uint32_t gso_types;
1423         uint16_t data_size;
1424         bool warning = 0;
1425         int k;
1426         int ret;
1427
1428         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1429
1430         /* Configuration of logical cores. */
1431         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1432                                 sizeof(struct fwd_lcore *) * nb_lcores,
1433                                 RTE_CACHE_LINE_SIZE);
1434         if (fwd_lcores == NULL) {
1435                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1436                                                         "failed\n", nb_lcores);
1437         }
1438         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1439                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1440                                                sizeof(struct fwd_lcore),
1441                                                RTE_CACHE_LINE_SIZE);
1442                 if (fwd_lcores[lc_id] == NULL) {
1443                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1444                                                                 "failed\n");
1445                 }
1446                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1447         }
1448
1449         RTE_ETH_FOREACH_DEV(pid) {
1450                 port = &ports[pid];
1451                 /* Apply default TxRx configuration for all ports */
1452                 port->dev_conf.txmode = tx_mode;
1453                 port->dev_conf.rxmode = rx_mode;
1454
1455                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1456                 if (ret != 0)
1457                         rte_exit(EXIT_FAILURE,
1458                                  "rte_eth_dev_info_get() failed\n");
1459
1460                 if (!(port->dev_info.tx_offload_capa &
1461                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1462                         port->dev_conf.txmode.offloads &=
1463                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1464                 if (numa_support) {
1465                         if (port_numa[pid] != NUMA_NO_CONFIG)
1466                                 port_per_socket[port_numa[pid]]++;
1467                         else {
1468                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1469
1470                                 /*
1471                                  * if socket_id is invalid,
1472                                  * set to the first available socket.
1473                                  */
1474                                 if (check_socket_id(socket_id) < 0)
1475                                         socket_id = socket_ids[0];
1476                                 port_per_socket[socket_id]++;
1477                         }
1478                 }
1479
1480                 /* Apply Rx offloads configuration */
1481                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1482                         port->rx_conf[k].offloads =
1483                                 port->dev_conf.rxmode.offloads;
1484                 /* Apply Tx offloads configuration */
1485                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1486                         port->tx_conf[k].offloads =
1487                                 port->dev_conf.txmode.offloads;
1488
1489                 /* set flag to initialize port/queue */
1490                 port->need_reconfig = 1;
1491                 port->need_reconfig_queues = 1;
1492                 port->tx_metadata = 0;
1493
1494                 /* Check for maximum number of segments per MTU. Accordingly
1495                  * update the mbuf data size.
1496                  */
1497                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1498                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1499                         data_size = rx_mode.max_rx_pkt_len /
1500                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1501
1502                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1503                                                         mbuf_data_size[0]) {
1504                                 mbuf_data_size[0] = data_size +
1505                                                  RTE_PKTMBUF_HEADROOM;
1506                                 warning = 1;
1507                         }
1508                 }
1509         }
1510
1511         if (warning)
1512                 TESTPMD_LOG(WARNING,
1513                             "Configured mbuf size of the first segment %hu\n",
1514                             mbuf_data_size[0]);
1515         /*
1516          * Create pools of mbuf.
1517          * If NUMA support is disabled, create a single pool of mbuf in
1518          * socket 0 memory by default.
1519          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1520          *
1521          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1522          * nb_txd can be configured at run time.
1523          */
1524         if (param_total_num_mbufs)
1525                 nb_mbuf_per_pool = param_total_num_mbufs;
1526         else {
1527                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1528                         (nb_lcores * mb_mempool_cache) +
1529                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1530                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1531         }
1532
1533         if (numa_support) {
1534                 uint8_t i, j;
1535
1536                 for (i = 0; i < num_sockets; i++)
1537                         for (j = 0; j < mbuf_data_size_n; j++)
1538                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1539                                         mbuf_pool_create(mbuf_data_size[j],
1540                                                           nb_mbuf_per_pool,
1541                                                           socket_ids[i], j);
1542         } else {
1543                 uint8_t i;
1544
1545                 for (i = 0; i < mbuf_data_size_n; i++)
1546                         mempools[i] = mbuf_pool_create
1547                                         (mbuf_data_size[i],
1548                                          nb_mbuf_per_pool,
1549                                          socket_num == UMA_NO_CONFIG ?
1550                                          0 : socket_num, i);
1551         }
1552
1553         init_port_config();
1554
1555         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1556                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1557         /*
1558          * Records which Mbuf pool to use by each logical core, if needed.
1559          */
1560         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1561                 mbp = mbuf_pool_find(
1562                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1563
1564                 if (mbp == NULL)
1565                         mbp = mbuf_pool_find(0, 0);
1566                 fwd_lcores[lc_id]->mbp = mbp;
1567                 /* initialize GSO context */
1568                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1569                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1570                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1571                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1572                         RTE_ETHER_CRC_LEN;
1573                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1574         }
1575
1576         /* Configuration of packet forwarding streams. */
1577         if (init_fwd_streams() < 0)
1578                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1579
1580         fwd_config_setup();
1581
1582         /* create a gro context for each lcore */
1583         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1584         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1585         gro_param.max_item_per_flow = MAX_PKT_BURST;
1586         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1587                 gro_param.socket_id = rte_lcore_to_socket_id(
1588                                 fwd_lcores_cpuids[lc_id]);
1589                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1590                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1591                         rte_exit(EXIT_FAILURE,
1592                                         "rte_gro_ctx_create() failed\n");
1593                 }
1594         }
1595 }
1596
1597
1598 void
1599 reconfig(portid_t new_port_id, unsigned socket_id)
1600 {
1601         struct rte_port *port;
1602         int ret;
1603
1604         /* Reconfiguration of Ethernet ports. */
1605         port = &ports[new_port_id];
1606
1607         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1608         if (ret != 0)
1609                 return;
1610
1611         /* set flag to initialize port/queue */
1612         port->need_reconfig = 1;
1613         port->need_reconfig_queues = 1;
1614         port->socket_id = socket_id;
1615
1616         init_port_config();
1617 }
1618
1619
1620 int
1621 init_fwd_streams(void)
1622 {
1623         portid_t pid;
1624         struct rte_port *port;
1625         streamid_t sm_id, nb_fwd_streams_new;
1626         queueid_t q;
1627
1628         /* set socket id according to numa or not */
1629         RTE_ETH_FOREACH_DEV(pid) {
1630                 port = &ports[pid];
1631                 if (nb_rxq > port->dev_info.max_rx_queues) {
1632                         printf("Fail: nb_rxq(%d) is greater than "
1633                                 "max_rx_queues(%d)\n", nb_rxq,
1634                                 port->dev_info.max_rx_queues);
1635                         return -1;
1636                 }
1637                 if (nb_txq > port->dev_info.max_tx_queues) {
1638                         printf("Fail: nb_txq(%d) is greater than "
1639                                 "max_tx_queues(%d)\n", nb_txq,
1640                                 port->dev_info.max_tx_queues);
1641                         return -1;
1642                 }
1643                 if (numa_support) {
1644                         if (port_numa[pid] != NUMA_NO_CONFIG)
1645                                 port->socket_id = port_numa[pid];
1646                         else {
1647                                 port->socket_id = rte_eth_dev_socket_id(pid);
1648
1649                                 /*
1650                                  * if socket_id is invalid,
1651                                  * set to the first available socket.
1652                                  */
1653                                 if (check_socket_id(port->socket_id) < 0)
1654                                         port->socket_id = socket_ids[0];
1655                         }
1656                 }
1657                 else {
1658                         if (socket_num == UMA_NO_CONFIG)
1659                                 port->socket_id = 0;
1660                         else
1661                                 port->socket_id = socket_num;
1662                 }
1663         }
1664
1665         q = RTE_MAX(nb_rxq, nb_txq);
1666         if (q == 0) {
1667                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1668                 return -1;
1669         }
1670         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1671         if (nb_fwd_streams_new == nb_fwd_streams)
1672                 return 0;
1673         /* clear the old */
1674         if (fwd_streams != NULL) {
1675                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1676                         if (fwd_streams[sm_id] == NULL)
1677                                 continue;
1678                         rte_free(fwd_streams[sm_id]);
1679                         fwd_streams[sm_id] = NULL;
1680                 }
1681                 rte_free(fwd_streams);
1682                 fwd_streams = NULL;
1683         }
1684
1685         /* init new */
1686         nb_fwd_streams = nb_fwd_streams_new;
1687         if (nb_fwd_streams) {
1688                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1689                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1690                         RTE_CACHE_LINE_SIZE);
1691                 if (fwd_streams == NULL)
1692                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1693                                  " (struct fwd_stream *)) failed\n",
1694                                  nb_fwd_streams);
1695
1696                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1697                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1698                                 " struct fwd_stream", sizeof(struct fwd_stream),
1699                                 RTE_CACHE_LINE_SIZE);
1700                         if (fwd_streams[sm_id] == NULL)
1701                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1702                                          "(struct fwd_stream) failed\n");
1703                 }
1704         }
1705
1706         return 0;
1707 }
1708
1709 static void
1710 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1711 {
1712         uint64_t total_burst, sburst;
1713         uint64_t nb_burst;
1714         uint64_t burst_stats[4];
1715         uint16_t pktnb_stats[4];
1716         uint16_t nb_pkt;
1717         int burst_percent[4], sburstp;
1718         int i;
1719
1720         /*
1721          * First compute the total number of packet bursts and the
1722          * two highest numbers of bursts of the same number of packets.
1723          */
1724         memset(&burst_stats, 0x0, sizeof(burst_stats));
1725         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1726
1727         /* Show stats for 0 burst size always */
1728         total_burst = pbs->pkt_burst_spread[0];
1729         burst_stats[0] = pbs->pkt_burst_spread[0];
1730         pktnb_stats[0] = 0;
1731
1732         /* Find the next 2 burst sizes with highest occurrences. */
1733         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1734                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1735
1736                 if (nb_burst == 0)
1737                         continue;
1738
1739                 total_burst += nb_burst;
1740
1741                 if (nb_burst > burst_stats[1]) {
1742                         burst_stats[2] = burst_stats[1];
1743                         pktnb_stats[2] = pktnb_stats[1];
1744                         burst_stats[1] = nb_burst;
1745                         pktnb_stats[1] = nb_pkt;
1746                 } else if (nb_burst > burst_stats[2]) {
1747                         burst_stats[2] = nb_burst;
1748                         pktnb_stats[2] = nb_pkt;
1749                 }
1750         }
1751         if (total_burst == 0)
1752                 return;
1753
1754         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1755         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1756                 if (i == 3) {
1757                         printf("%d%% of other]\n", 100 - sburstp);
1758                         return;
1759                 }
1760
1761                 sburst += burst_stats[i];
1762                 if (sburst == total_burst) {
1763                         printf("%d%% of %d pkts]\n",
1764                                 100 - sburstp, (int) pktnb_stats[i]);
1765                         return;
1766                 }
1767
1768                 burst_percent[i] =
1769                         (double)burst_stats[i] / total_burst * 100;
1770                 printf("%d%% of %d pkts + ",
1771                         burst_percent[i], (int) pktnb_stats[i]);
1772                 sburstp += burst_percent[i];
1773         }
1774 }
1775
1776 static void
1777 fwd_stream_stats_display(streamid_t stream_id)
1778 {
1779         struct fwd_stream *fs;
1780         static const char *fwd_top_stats_border = "-------";
1781
1782         fs = fwd_streams[stream_id];
1783         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1784             (fs->fwd_dropped == 0))
1785                 return;
1786         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1787                "TX Port=%2d/Queue=%2d %s\n",
1788                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1789                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1790         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1791                " TX-dropped: %-14"PRIu64,
1792                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1793
1794         /* if checksum mode */
1795         if (cur_fwd_eng == &csum_fwd_engine) {
1796                 printf("  RX- bad IP checksum: %-14"PRIu64
1797                        "  Rx- bad L4 checksum: %-14"PRIu64
1798                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1799                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1800                         fs->rx_bad_outer_l4_csum);
1801         } else {
1802                 printf("\n");
1803         }
1804
1805         if (record_burst_stats) {
1806                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1807                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1808         }
1809 }
1810
1811 void
1812 fwd_stats_display(void)
1813 {
1814         static const char *fwd_stats_border = "----------------------";
1815         static const char *acc_stats_border = "+++++++++++++++";
1816         struct {
1817                 struct fwd_stream *rx_stream;
1818                 struct fwd_stream *tx_stream;
1819                 uint64_t tx_dropped;
1820                 uint64_t rx_bad_ip_csum;
1821                 uint64_t rx_bad_l4_csum;
1822                 uint64_t rx_bad_outer_l4_csum;
1823         } ports_stats[RTE_MAX_ETHPORTS];
1824         uint64_t total_rx_dropped = 0;
1825         uint64_t total_tx_dropped = 0;
1826         uint64_t total_rx_nombuf = 0;
1827         struct rte_eth_stats stats;
1828         uint64_t fwd_cycles = 0;
1829         uint64_t total_recv = 0;
1830         uint64_t total_xmit = 0;
1831         struct rte_port *port;
1832         streamid_t sm_id;
1833         portid_t pt_id;
1834         int i;
1835
1836         memset(ports_stats, 0, sizeof(ports_stats));
1837
1838         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1839                 struct fwd_stream *fs = fwd_streams[sm_id];
1840
1841                 if (cur_fwd_config.nb_fwd_streams >
1842                     cur_fwd_config.nb_fwd_ports) {
1843                         fwd_stream_stats_display(sm_id);
1844                 } else {
1845                         ports_stats[fs->tx_port].tx_stream = fs;
1846                         ports_stats[fs->rx_port].rx_stream = fs;
1847                 }
1848
1849                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1850
1851                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1852                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1853                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1854                                 fs->rx_bad_outer_l4_csum;
1855
1856                 if (record_core_cycles)
1857                         fwd_cycles += fs->core_cycles;
1858         }
1859         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1860                 uint8_t j;
1861
1862                 pt_id = fwd_ports_ids[i];
1863                 port = &ports[pt_id];
1864
1865                 rte_eth_stats_get(pt_id, &stats);
1866                 stats.ipackets -= port->stats.ipackets;
1867                 stats.opackets -= port->stats.opackets;
1868                 stats.ibytes -= port->stats.ibytes;
1869                 stats.obytes -= port->stats.obytes;
1870                 stats.imissed -= port->stats.imissed;
1871                 stats.oerrors -= port->stats.oerrors;
1872                 stats.rx_nombuf -= port->stats.rx_nombuf;
1873
1874                 total_recv += stats.ipackets;
1875                 total_xmit += stats.opackets;
1876                 total_rx_dropped += stats.imissed;
1877                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1878                 total_tx_dropped += stats.oerrors;
1879                 total_rx_nombuf  += stats.rx_nombuf;
1880
1881                 printf("\n  %s Forward statistics for port %-2d %s\n",
1882                        fwd_stats_border, pt_id, fwd_stats_border);
1883
1884                 if (!port->rx_queue_stats_mapping_enabled &&
1885                     !port->tx_queue_stats_mapping_enabled) {
1886                         printf("  RX-packets: %-14"PRIu64
1887                                " RX-dropped: %-14"PRIu64
1888                                "RX-total: %-"PRIu64"\n",
1889                                stats.ipackets, stats.imissed,
1890                                stats.ipackets + stats.imissed);
1891
1892                         if (cur_fwd_eng == &csum_fwd_engine)
1893                                 printf("  Bad-ipcsum: %-14"PRIu64
1894                                        " Bad-l4csum: %-14"PRIu64
1895                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1896                                        ports_stats[pt_id].rx_bad_ip_csum,
1897                                        ports_stats[pt_id].rx_bad_l4_csum,
1898                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1899                         if (stats.ierrors + stats.rx_nombuf > 0) {
1900                                 printf("  RX-error: %-"PRIu64"\n",
1901                                        stats.ierrors);
1902                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1903                                        stats.rx_nombuf);
1904                         }
1905
1906                         printf("  TX-packets: %-14"PRIu64
1907                                " TX-dropped: %-14"PRIu64
1908                                "TX-total: %-"PRIu64"\n",
1909                                stats.opackets, ports_stats[pt_id].tx_dropped,
1910                                stats.opackets + ports_stats[pt_id].tx_dropped);
1911                 } else {
1912                         printf("  RX-packets:             %14"PRIu64
1913                                "    RX-dropped:%14"PRIu64
1914                                "    RX-total:%14"PRIu64"\n",
1915                                stats.ipackets, stats.imissed,
1916                                stats.ipackets + stats.imissed);
1917
1918                         if (cur_fwd_eng == &csum_fwd_engine)
1919                                 printf("  Bad-ipcsum:%14"PRIu64
1920                                        "    Bad-l4csum:%14"PRIu64
1921                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1922                                        ports_stats[pt_id].rx_bad_ip_csum,
1923                                        ports_stats[pt_id].rx_bad_l4_csum,
1924                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1925                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1926                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1927                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1928                                        stats.rx_nombuf);
1929                         }
1930
1931                         printf("  TX-packets:             %14"PRIu64
1932                                "    TX-dropped:%14"PRIu64
1933                                "    TX-total:%14"PRIu64"\n",
1934                                stats.opackets, ports_stats[pt_id].tx_dropped,
1935                                stats.opackets + ports_stats[pt_id].tx_dropped);
1936                 }
1937
1938                 if (record_burst_stats) {
1939                         if (ports_stats[pt_id].rx_stream)
1940                                 pkt_burst_stats_display("RX",
1941                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1942                         if (ports_stats[pt_id].tx_stream)
1943                                 pkt_burst_stats_display("TX",
1944                                         &ports_stats[pt_id].tx_stream->tx_burst_stats);
1945                 }
1946
1947                 if (port->rx_queue_stats_mapping_enabled) {
1948                         printf("\n");
1949                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1950                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1951                                        "     RX-errors:%14"PRIu64
1952                                        "    RX-bytes:%14"PRIu64"\n",
1953                                        j, stats.q_ipackets[j],
1954                                        stats.q_errors[j], stats.q_ibytes[j]);
1955                         }
1956                         printf("\n");
1957                 }
1958                 if (port->tx_queue_stats_mapping_enabled) {
1959                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1960                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1961                                        "                                 TX-bytes:%14"
1962                                        PRIu64"\n",
1963                                        j, stats.q_opackets[j],
1964                                        stats.q_obytes[j]);
1965                         }
1966                 }
1967
1968                 printf("  %s--------------------------------%s\n",
1969                        fwd_stats_border, fwd_stats_border);
1970         }
1971
1972         printf("\n  %s Accumulated forward statistics for all ports"
1973                "%s\n",
1974                acc_stats_border, acc_stats_border);
1975         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1976                "%-"PRIu64"\n"
1977                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1978                "%-"PRIu64"\n",
1979                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1980                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1981         if (total_rx_nombuf > 0)
1982                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1983         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1984                "%s\n",
1985                acc_stats_border, acc_stats_border);
1986         if (record_core_cycles) {
1987 #define CYC_PER_MHZ 1E6
1988                 if (total_recv > 0 || total_xmit > 0) {
1989                         uint64_t total_pkts = 0;
1990                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1991                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1992                                 total_pkts = total_xmit;
1993                         else
1994                                 total_pkts = total_recv;
1995
1996                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1997                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1998                                " MHz Clock\n",
1999                                (double) fwd_cycles / total_pkts,
2000                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2001                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2002                 }
2003         }
2004 }
2005
2006 void
2007 fwd_stats_reset(void)
2008 {
2009         streamid_t sm_id;
2010         portid_t pt_id;
2011         int i;
2012
2013         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2014                 pt_id = fwd_ports_ids[i];
2015                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2016         }
2017         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2018                 struct fwd_stream *fs = fwd_streams[sm_id];
2019
2020                 fs->rx_packets = 0;
2021                 fs->tx_packets = 0;
2022                 fs->fwd_dropped = 0;
2023                 fs->rx_bad_ip_csum = 0;
2024                 fs->rx_bad_l4_csum = 0;
2025                 fs->rx_bad_outer_l4_csum = 0;
2026
2027                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2028                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2029                 fs->core_cycles = 0;
2030         }
2031 }
2032
2033 static void
2034 flush_fwd_rx_queues(void)
2035 {
2036         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2037         portid_t  rxp;
2038         portid_t port_id;
2039         queueid_t rxq;
2040         uint16_t  nb_rx;
2041         uint16_t  i;
2042         uint8_t   j;
2043         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2044         uint64_t timer_period;
2045
2046         /* convert to number of cycles */
2047         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2048
2049         for (j = 0; j < 2; j++) {
2050                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2051                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2052                                 port_id = fwd_ports_ids[rxp];
2053                                 /**
2054                                 * testpmd can stuck in the below do while loop
2055                                 * if rte_eth_rx_burst() always returns nonzero
2056                                 * packets. So timer is added to exit this loop
2057                                 * after 1sec timer expiry.
2058                                 */
2059                                 prev_tsc = rte_rdtsc();
2060                                 do {
2061                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2062                                                 pkts_burst, MAX_PKT_BURST);
2063                                         for (i = 0; i < nb_rx; i++)
2064                                                 rte_pktmbuf_free(pkts_burst[i]);
2065
2066                                         cur_tsc = rte_rdtsc();
2067                                         diff_tsc = cur_tsc - prev_tsc;
2068                                         timer_tsc += diff_tsc;
2069                                 } while ((nb_rx > 0) &&
2070                                         (timer_tsc < timer_period));
2071                                 timer_tsc = 0;
2072                         }
2073                 }
2074                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2075         }
2076 }
2077
2078 static void
2079 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2080 {
2081         struct fwd_stream **fsm;
2082         streamid_t nb_fs;
2083         streamid_t sm_id;
2084 #ifdef RTE_LIBRTE_BITRATESTATS
2085         uint64_t tics_per_1sec;
2086         uint64_t tics_datum;
2087         uint64_t tics_current;
2088         uint16_t i, cnt_ports;
2089
2090         cnt_ports = nb_ports;
2091         tics_datum = rte_rdtsc();
2092         tics_per_1sec = rte_get_timer_hz();
2093 #endif
2094         fsm = &fwd_streams[fc->stream_idx];
2095         nb_fs = fc->stream_nb;
2096         do {
2097                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2098                         (*pkt_fwd)(fsm[sm_id]);
2099 #ifdef RTE_LIBRTE_BITRATESTATS
2100                 if (bitrate_enabled != 0 &&
2101                                 bitrate_lcore_id == rte_lcore_id()) {
2102                         tics_current = rte_rdtsc();
2103                         if (tics_current - tics_datum >= tics_per_1sec) {
2104                                 /* Periodic bitrate calculation */
2105                                 for (i = 0; i < cnt_ports; i++)
2106                                         rte_stats_bitrate_calc(bitrate_data,
2107                                                 ports_ids[i]);
2108                                 tics_datum = tics_current;
2109                         }
2110                 }
2111 #endif
2112 #ifdef RTE_LIBRTE_LATENCY_STATS
2113                 if (latencystats_enabled != 0 &&
2114                                 latencystats_lcore_id == rte_lcore_id())
2115                         rte_latencystats_update();
2116 #endif
2117
2118         } while (! fc->stopped);
2119 }
2120
2121 static int
2122 start_pkt_forward_on_core(void *fwd_arg)
2123 {
2124         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2125                              cur_fwd_config.fwd_eng->packet_fwd);
2126         return 0;
2127 }
2128
2129 /*
2130  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2131  * Used to start communication flows in network loopback test configurations.
2132  */
2133 static int
2134 run_one_txonly_burst_on_core(void *fwd_arg)
2135 {
2136         struct fwd_lcore *fwd_lc;
2137         struct fwd_lcore tmp_lcore;
2138
2139         fwd_lc = (struct fwd_lcore *) fwd_arg;
2140         tmp_lcore = *fwd_lc;
2141         tmp_lcore.stopped = 1;
2142         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2143         return 0;
2144 }
2145
2146 /*
2147  * Launch packet forwarding:
2148  *     - Setup per-port forwarding context.
2149  *     - launch logical cores with their forwarding configuration.
2150  */
2151 static void
2152 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2153 {
2154         port_fwd_begin_t port_fwd_begin;
2155         unsigned int i;
2156         unsigned int lc_id;
2157         int diag;
2158
2159         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2160         if (port_fwd_begin != NULL) {
2161                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2162                         (*port_fwd_begin)(fwd_ports_ids[i]);
2163         }
2164         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2165                 lc_id = fwd_lcores_cpuids[i];
2166                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2167                         fwd_lcores[i]->stopped = 0;
2168                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2169                                                      fwd_lcores[i], lc_id);
2170                         if (diag != 0)
2171                                 printf("launch lcore %u failed - diag=%d\n",
2172                                        lc_id, diag);
2173                 }
2174         }
2175 }
2176
2177 /*
2178  * Launch packet forwarding configuration.
2179  */
2180 void
2181 start_packet_forwarding(int with_tx_first)
2182 {
2183         port_fwd_begin_t port_fwd_begin;
2184         port_fwd_end_t  port_fwd_end;
2185         struct rte_port *port;
2186         unsigned int i;
2187         portid_t   pt_id;
2188
2189         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2190                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2191
2192         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2193                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2194
2195         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2196                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2197                 (!nb_rxq || !nb_txq))
2198                 rte_exit(EXIT_FAILURE,
2199                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2200                         cur_fwd_eng->fwd_mode_name);
2201
2202         if (all_ports_started() == 0) {
2203                 printf("Not all ports were started\n");
2204                 return;
2205         }
2206         if (test_done == 0) {
2207                 printf("Packet forwarding already started\n");
2208                 return;
2209         }
2210
2211
2212         if(dcb_test) {
2213                 for (i = 0; i < nb_fwd_ports; i++) {
2214                         pt_id = fwd_ports_ids[i];
2215                         port = &ports[pt_id];
2216                         if (!port->dcb_flag) {
2217                                 printf("In DCB mode, all forwarding ports must "
2218                                        "be configured in this mode.\n");
2219                                 return;
2220                         }
2221                 }
2222                 if (nb_fwd_lcores == 1) {
2223                         printf("In DCB mode,the nb forwarding cores "
2224                                "should be larger than 1.\n");
2225                         return;
2226                 }
2227         }
2228         test_done = 0;
2229
2230         fwd_config_setup();
2231
2232         if(!no_flush_rx)
2233                 flush_fwd_rx_queues();
2234
2235         pkt_fwd_config_display(&cur_fwd_config);
2236         rxtx_config_display();
2237
2238         fwd_stats_reset();
2239         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2240                 pt_id = fwd_ports_ids[i];
2241                 port = &ports[pt_id];
2242                 map_port_queue_stats_mapping_registers(pt_id, port);
2243         }
2244         if (with_tx_first) {
2245                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2246                 if (port_fwd_begin != NULL) {
2247                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2248                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2249                 }
2250                 while (with_tx_first--) {
2251                         launch_packet_forwarding(
2252                                         run_one_txonly_burst_on_core);
2253                         rte_eal_mp_wait_lcore();
2254                 }
2255                 port_fwd_end = tx_only_engine.port_fwd_end;
2256                 if (port_fwd_end != NULL) {
2257                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2258                                 (*port_fwd_end)(fwd_ports_ids[i]);
2259                 }
2260         }
2261         launch_packet_forwarding(start_pkt_forward_on_core);
2262 }
2263
2264 void
2265 stop_packet_forwarding(void)
2266 {
2267         port_fwd_end_t port_fwd_end;
2268         lcoreid_t lc_id;
2269         portid_t pt_id;
2270         int i;
2271
2272         if (test_done) {
2273                 printf("Packet forwarding not started\n");
2274                 return;
2275         }
2276         printf("Telling cores to stop...");
2277         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2278                 fwd_lcores[lc_id]->stopped = 1;
2279         printf("\nWaiting for lcores to finish...\n");
2280         rte_eal_mp_wait_lcore();
2281         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2282         if (port_fwd_end != NULL) {
2283                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2284                         pt_id = fwd_ports_ids[i];
2285                         (*port_fwd_end)(pt_id);
2286                 }
2287         }
2288
2289         fwd_stats_display();
2290
2291         printf("\nDone.\n");
2292         test_done = 1;
2293 }
2294
2295 void
2296 dev_set_link_up(portid_t pid)
2297 {
2298         if (rte_eth_dev_set_link_up(pid) < 0)
2299                 printf("\nSet link up fail.\n");
2300 }
2301
2302 void
2303 dev_set_link_down(portid_t pid)
2304 {
2305         if (rte_eth_dev_set_link_down(pid) < 0)
2306                 printf("\nSet link down fail.\n");
2307 }
2308
2309 static int
2310 all_ports_started(void)
2311 {
2312         portid_t pi;
2313         struct rte_port *port;
2314
2315         RTE_ETH_FOREACH_DEV(pi) {
2316                 port = &ports[pi];
2317                 /* Check if there is a port which is not started */
2318                 if ((port->port_status != RTE_PORT_STARTED) &&
2319                         (port->slave_flag == 0))
2320                         return 0;
2321         }
2322
2323         /* No port is not started */
2324         return 1;
2325 }
2326
2327 int
2328 port_is_stopped(portid_t port_id)
2329 {
2330         struct rte_port *port = &ports[port_id];
2331
2332         if ((port->port_status != RTE_PORT_STOPPED) &&
2333             (port->slave_flag == 0))
2334                 return 0;
2335         return 1;
2336 }
2337
2338 int
2339 all_ports_stopped(void)
2340 {
2341         portid_t pi;
2342
2343         RTE_ETH_FOREACH_DEV(pi) {
2344                 if (!port_is_stopped(pi))
2345                         return 0;
2346         }
2347
2348         return 1;
2349 }
2350
2351 int
2352 port_is_started(portid_t port_id)
2353 {
2354         if (port_id_is_invalid(port_id, ENABLED_WARN))
2355                 return 0;
2356
2357         if (ports[port_id].port_status != RTE_PORT_STARTED)
2358                 return 0;
2359
2360         return 1;
2361 }
2362
2363 /* Configure the Rx and Tx hairpin queues for the selected port. */
2364 static int
2365 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2366 {
2367         queueid_t qi;
2368         struct rte_eth_hairpin_conf hairpin_conf = {
2369                 .peer_count = 1,
2370         };
2371         int i;
2372         int diag;
2373         struct rte_port *port = &ports[pi];
2374         uint16_t peer_rx_port = pi;
2375         uint16_t peer_tx_port = pi;
2376         uint32_t manual = 1;
2377         uint32_t tx_exp = hairpin_mode & 0x10;
2378
2379         if (!(hairpin_mode & 0xf)) {
2380                 peer_rx_port = pi;
2381                 peer_tx_port = pi;
2382                 manual = 0;
2383         } else if (hairpin_mode & 0x1) {
2384                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2385                                                        RTE_ETH_DEV_NO_OWNER);
2386                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2387                         peer_tx_port = rte_eth_find_next_owned_by(0,
2388                                                 RTE_ETH_DEV_NO_OWNER);
2389                 if (p_pi != RTE_MAX_ETHPORTS) {
2390                         peer_rx_port = p_pi;
2391                 } else {
2392                         uint16_t next_pi;
2393
2394                         /* Last port will be the peer RX port of the first. */
2395                         RTE_ETH_FOREACH_DEV(next_pi)
2396                                 peer_rx_port = next_pi;
2397                 }
2398                 manual = 1;
2399         } else if (hairpin_mode & 0x2) {
2400                 if (cnt_pi & 0x1) {
2401                         peer_rx_port = p_pi;
2402                 } else {
2403                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2404                                                 RTE_ETH_DEV_NO_OWNER);
2405                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2406                                 peer_rx_port = pi;
2407                 }
2408                 peer_tx_port = peer_rx_port;
2409                 manual = 1;
2410         }
2411
2412         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2413                 hairpin_conf.peers[0].port = peer_rx_port;
2414                 hairpin_conf.peers[0].queue = i + nb_rxq;
2415                 hairpin_conf.manual_bind = !!manual;
2416                 hairpin_conf.tx_explicit = !!tx_exp;
2417                 diag = rte_eth_tx_hairpin_queue_setup
2418                         (pi, qi, nb_txd, &hairpin_conf);
2419                 i++;
2420                 if (diag == 0)
2421                         continue;
2422
2423                 /* Fail to setup rx queue, return */
2424                 if (rte_atomic16_cmpset(&(port->port_status),
2425                                         RTE_PORT_HANDLING,
2426                                         RTE_PORT_STOPPED) == 0)
2427                         printf("Port %d can not be set back "
2428                                         "to stopped\n", pi);
2429                 printf("Fail to configure port %d hairpin "
2430                                 "queues\n", pi);
2431                 /* try to reconfigure queues next time */
2432                 port->need_reconfig_queues = 1;
2433                 return -1;
2434         }
2435         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2436                 hairpin_conf.peers[0].port = peer_tx_port;
2437                 hairpin_conf.peers[0].queue = i + nb_txq;
2438                 hairpin_conf.manual_bind = !!manual;
2439                 hairpin_conf.tx_explicit = !!tx_exp;
2440                 diag = rte_eth_rx_hairpin_queue_setup
2441                         (pi, qi, nb_rxd, &hairpin_conf);
2442                 i++;
2443                 if (diag == 0)
2444                         continue;
2445
2446                 /* Fail to setup rx queue, return */
2447                 if (rte_atomic16_cmpset(&(port->port_status),
2448                                         RTE_PORT_HANDLING,
2449                                         RTE_PORT_STOPPED) == 0)
2450                         printf("Port %d can not be set back "
2451                                         "to stopped\n", pi);
2452                 printf("Fail to configure port %d hairpin "
2453                                 "queues\n", pi);
2454                 /* try to reconfigure queues next time */
2455                 port->need_reconfig_queues = 1;
2456                 return -1;
2457         }
2458         return 0;
2459 }
2460
2461 int
2462 start_port(portid_t pid)
2463 {
2464         int diag, need_check_link_status = -1;
2465         portid_t pi;
2466         portid_t p_pi = RTE_MAX_ETHPORTS;
2467         portid_t pl[RTE_MAX_ETHPORTS];
2468         portid_t peer_pl[RTE_MAX_ETHPORTS];
2469         uint16_t cnt_pi = 0;
2470         uint16_t cfg_pi = 0;
2471         int peer_pi;
2472         queueid_t qi;
2473         struct rte_port *port;
2474         struct rte_ether_addr mac_addr;
2475         struct rte_eth_hairpin_cap cap;
2476
2477         if (port_id_is_invalid(pid, ENABLED_WARN))
2478                 return 0;
2479
2480         if(dcb_config)
2481                 dcb_test = 1;
2482         RTE_ETH_FOREACH_DEV(pi) {
2483                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2484                         continue;
2485
2486                 need_check_link_status = 0;
2487                 port = &ports[pi];
2488                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2489                                                  RTE_PORT_HANDLING) == 0) {
2490                         printf("Port %d is now not stopped\n", pi);
2491                         continue;
2492                 }
2493
2494                 if (port->need_reconfig > 0) {
2495                         port->need_reconfig = 0;
2496
2497                         if (flow_isolate_all) {
2498                                 int ret = port_flow_isolate(pi, 1);
2499                                 if (ret) {
2500                                         printf("Failed to apply isolated"
2501                                                " mode on port %d\n", pi);
2502                                         return -1;
2503                                 }
2504                         }
2505                         configure_rxtx_dump_callbacks(0);
2506                         printf("Configuring Port %d (socket %u)\n", pi,
2507                                         port->socket_id);
2508                         if (nb_hairpinq > 0 &&
2509                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2510                                 printf("Port %d doesn't support hairpin "
2511                                        "queues\n", pi);
2512                                 return -1;
2513                         }
2514                         /* configure port */
2515                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2516                                                      nb_txq + nb_hairpinq,
2517                                                      &(port->dev_conf));
2518                         if (diag != 0) {
2519                                 if (rte_atomic16_cmpset(&(port->port_status),
2520                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2521                                         printf("Port %d can not be set back "
2522                                                         "to stopped\n", pi);
2523                                 printf("Fail to configure port %d\n", pi);
2524                                 /* try to reconfigure port next time */
2525                                 port->need_reconfig = 1;
2526                                 return -1;
2527                         }
2528                 }
2529                 if (port->need_reconfig_queues > 0) {
2530                         port->need_reconfig_queues = 0;
2531                         /* setup tx queues */
2532                         for (qi = 0; qi < nb_txq; qi++) {
2533                                 if ((numa_support) &&
2534                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2535                                         diag = rte_eth_tx_queue_setup(pi, qi,
2536                                                 port->nb_tx_desc[qi],
2537                                                 txring_numa[pi],
2538                                                 &(port->tx_conf[qi]));
2539                                 else
2540                                         diag = rte_eth_tx_queue_setup(pi, qi,
2541                                                 port->nb_tx_desc[qi],
2542                                                 port->socket_id,
2543                                                 &(port->tx_conf[qi]));
2544
2545                                 if (diag == 0)
2546                                         continue;
2547
2548                                 /* Fail to setup tx queue, return */
2549                                 if (rte_atomic16_cmpset(&(port->port_status),
2550                                                         RTE_PORT_HANDLING,
2551                                                         RTE_PORT_STOPPED) == 0)
2552                                         printf("Port %d can not be set back "
2553                                                         "to stopped\n", pi);
2554                                 printf("Fail to configure port %d tx queues\n",
2555                                        pi);
2556                                 /* try to reconfigure queues next time */
2557                                 port->need_reconfig_queues = 1;
2558                                 return -1;
2559                         }
2560                         for (qi = 0; qi < nb_rxq; qi++) {
2561                                 /* setup rx queues */
2562                                 if ((numa_support) &&
2563                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2564                                         struct rte_mempool * mp =
2565                                                 mbuf_pool_find
2566                                                         (rxring_numa[pi], 0);
2567                                         if (mp == NULL) {
2568                                                 printf("Failed to setup RX queue:"
2569                                                         "No mempool allocation"
2570                                                         " on the socket %d\n",
2571                                                         rxring_numa[pi]);
2572                                                 return -1;
2573                                         }
2574
2575                                         diag = rte_eth_rx_queue_setup(pi, qi,
2576                                              port->nb_rx_desc[qi],
2577                                              rxring_numa[pi],
2578                                              &(port->rx_conf[qi]),
2579                                              mp);
2580                                 } else {
2581                                         struct rte_mempool *mp =
2582                                                 mbuf_pool_find
2583                                                         (port->socket_id, 0);
2584                                         if (mp == NULL) {
2585                                                 printf("Failed to setup RX queue:"
2586                                                         "No mempool allocation"
2587                                                         " on the socket %d\n",
2588                                                         port->socket_id);
2589                                                 return -1;
2590                                         }
2591                                         diag = rte_eth_rx_queue_setup(pi, qi,
2592                                              port->nb_rx_desc[qi],
2593                                              port->socket_id,
2594                                              &(port->rx_conf[qi]),
2595                                              mp);
2596                                 }
2597                                 if (diag == 0)
2598                                         continue;
2599
2600                                 /* Fail to setup rx queue, return */
2601                                 if (rte_atomic16_cmpset(&(port->port_status),
2602                                                         RTE_PORT_HANDLING,
2603                                                         RTE_PORT_STOPPED) == 0)
2604                                         printf("Port %d can not be set back "
2605                                                         "to stopped\n", pi);
2606                                 printf("Fail to configure port %d rx queues\n",
2607                                        pi);
2608                                 /* try to reconfigure queues next time */
2609                                 port->need_reconfig_queues = 1;
2610                                 return -1;
2611                         }
2612                         /* setup hairpin queues */
2613                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2614                                 return -1;
2615                 }
2616                 configure_rxtx_dump_callbacks(verbose_level);
2617                 if (clear_ptypes) {
2618                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2619                                         NULL, 0);
2620                         if (diag < 0)
2621                                 printf(
2622                                 "Port %d: Failed to disable Ptype parsing\n",
2623                                 pi);
2624                 }
2625
2626                 p_pi = pi;
2627                 cnt_pi++;
2628
2629                 /* start port */
2630                 if (rte_eth_dev_start(pi) < 0) {
2631                         printf("Fail to start port %d\n", pi);
2632
2633                         /* Fail to setup rx queue, return */
2634                         if (rte_atomic16_cmpset(&(port->port_status),
2635                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2636                                 printf("Port %d can not be set back to "
2637                                                         "stopped\n", pi);
2638                         continue;
2639                 }
2640
2641                 if (rte_atomic16_cmpset(&(port->port_status),
2642                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2643                         printf("Port %d can not be set into started\n", pi);
2644
2645                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2646                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2647                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2648                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2649                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2650
2651                 /* at least one port started, need checking link status */
2652                 need_check_link_status = 1;
2653
2654                 pl[cfg_pi++] = pi;
2655         }
2656
2657         if (need_check_link_status == 1 && !no_link_check)
2658                 check_all_ports_link_status(RTE_PORT_ALL);
2659         else if (need_check_link_status == 0)
2660                 printf("Please stop the ports first\n");
2661
2662         if (hairpin_mode & 0xf) {
2663                 uint16_t i;
2664                 int j;
2665
2666                 /* bind all started hairpin ports */
2667                 for (i = 0; i < cfg_pi; i++) {
2668                         pi = pl[i];
2669                         /* bind current Tx to all peer Rx */
2670                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2671                                                         RTE_MAX_ETHPORTS, 1);
2672                         if (peer_pi < 0)
2673                                 return peer_pi;
2674                         for (j = 0; j < peer_pi; j++) {
2675                                 if (!port_is_started(peer_pl[j]))
2676                                         continue;
2677                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2678                                 if (diag < 0) {
2679                                         printf("Error during binding hairpin"
2680                                                " Tx port %u to %u: %s\n",
2681                                                pi, peer_pl[j],
2682                                                rte_strerror(-diag));
2683                                         return -1;
2684                                 }
2685                         }
2686                         /* bind all peer Tx to current Rx */
2687                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2688                                                         RTE_MAX_ETHPORTS, 0);
2689                         if (peer_pi < 0)
2690                                 return peer_pi;
2691                         for (j = 0; j < peer_pi; j++) {
2692                                 if (!port_is_started(peer_pl[j]))
2693                                         continue;
2694                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2695                                 if (diag < 0) {
2696                                         printf("Error during binding hairpin"
2697                                                " Tx port %u to %u: %s\n",
2698                                                peer_pl[j], pi,
2699                                                rte_strerror(-diag));
2700                                         return -1;
2701                                 }
2702                         }
2703                 }
2704         }
2705
2706         printf("Done\n");
2707         return 0;
2708 }
2709
2710 void
2711 stop_port(portid_t pid)
2712 {
2713         portid_t pi;
2714         struct rte_port *port;
2715         int need_check_link_status = 0;
2716         portid_t peer_pl[RTE_MAX_ETHPORTS];
2717         int peer_pi;
2718
2719         if (dcb_test) {
2720                 dcb_test = 0;
2721                 dcb_config = 0;
2722         }
2723
2724         if (port_id_is_invalid(pid, ENABLED_WARN))
2725                 return;
2726
2727         printf("Stopping ports...\n");
2728
2729         RTE_ETH_FOREACH_DEV(pi) {
2730                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2731                         continue;
2732
2733                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2734                         printf("Please remove port %d from forwarding configuration.\n", pi);
2735                         continue;
2736                 }
2737
2738                 if (port_is_bonding_slave(pi)) {
2739                         printf("Please remove port %d from bonded device.\n", pi);
2740                         continue;
2741                 }
2742
2743                 port = &ports[pi];
2744                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2745                                                 RTE_PORT_HANDLING) == 0)
2746                         continue;
2747
2748                 if (hairpin_mode & 0xf) {
2749                         int j;
2750
2751                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2752                         /* unbind all peer Tx from current Rx */
2753                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2754                                                         RTE_MAX_ETHPORTS, 0);
2755                         if (peer_pi < 0)
2756                                 continue;
2757                         for (j = 0; j < peer_pi; j++) {
2758                                 if (!port_is_started(peer_pl[j]))
2759                                         continue;
2760                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2761                         }
2762                 }
2763
2764                 rte_eth_dev_stop(pi);
2765
2766                 if (rte_atomic16_cmpset(&(port->port_status),
2767                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2768                         printf("Port %d can not be set into stopped\n", pi);
2769                 need_check_link_status = 1;
2770         }
2771         if (need_check_link_status && !no_link_check)
2772                 check_all_ports_link_status(RTE_PORT_ALL);
2773
2774         printf("Done\n");
2775 }
2776
2777 static void
2778 remove_invalid_ports_in(portid_t *array, portid_t *total)
2779 {
2780         portid_t i;
2781         portid_t new_total = 0;
2782
2783         for (i = 0; i < *total; i++)
2784                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2785                         array[new_total] = array[i];
2786                         new_total++;
2787                 }
2788         *total = new_total;
2789 }
2790
2791 static void
2792 remove_invalid_ports(void)
2793 {
2794         remove_invalid_ports_in(ports_ids, &nb_ports);
2795         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2796         nb_cfg_ports = nb_fwd_ports;
2797 }
2798
2799 void
2800 close_port(portid_t pid)
2801 {
2802         portid_t pi;
2803         struct rte_port *port;
2804
2805         if (port_id_is_invalid(pid, ENABLED_WARN))
2806                 return;
2807
2808         printf("Closing ports...\n");
2809
2810         RTE_ETH_FOREACH_DEV(pi) {
2811                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2812                         continue;
2813
2814                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2815                         printf("Please remove port %d from forwarding configuration.\n", pi);
2816                         continue;
2817                 }
2818
2819                 if (port_is_bonding_slave(pi)) {
2820                         printf("Please remove port %d from bonded device.\n", pi);
2821                         continue;
2822                 }
2823
2824                 port = &ports[pi];
2825                 if (rte_atomic16_cmpset(&(port->port_status),
2826                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2827                         printf("Port %d is already closed\n", pi);
2828                         continue;
2829                 }
2830
2831                 port_flow_flush(pi);
2832                 rte_eth_dev_close(pi);
2833         }
2834
2835         remove_invalid_ports();
2836         printf("Done\n");
2837 }
2838
2839 void
2840 reset_port(portid_t pid)
2841 {
2842         int diag;
2843         portid_t pi;
2844         struct rte_port *port;
2845
2846         if (port_id_is_invalid(pid, ENABLED_WARN))
2847                 return;
2848
2849         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2850                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2851                 printf("Can not reset port(s), please stop port(s) first.\n");
2852                 return;
2853         }
2854
2855         printf("Resetting ports...\n");
2856
2857         RTE_ETH_FOREACH_DEV(pi) {
2858                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2859                         continue;
2860
2861                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2862                         printf("Please remove port %d from forwarding "
2863                                "configuration.\n", pi);
2864                         continue;
2865                 }
2866
2867                 if (port_is_bonding_slave(pi)) {
2868                         printf("Please remove port %d from bonded device.\n",
2869                                pi);
2870                         continue;
2871                 }
2872
2873                 diag = rte_eth_dev_reset(pi);
2874                 if (diag == 0) {
2875                         port = &ports[pi];
2876                         port->need_reconfig = 1;
2877                         port->need_reconfig_queues = 1;
2878                 } else {
2879                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2880                 }
2881         }
2882
2883         printf("Done\n");
2884 }
2885
2886 void
2887 attach_port(char *identifier)
2888 {
2889         portid_t pi;
2890         struct rte_dev_iterator iterator;
2891
2892         printf("Attaching a new port...\n");
2893
2894         if (identifier == NULL) {
2895                 printf("Invalid parameters are specified\n");
2896                 return;
2897         }
2898
2899         if (rte_dev_probe(identifier) < 0) {
2900                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2901                 return;
2902         }
2903
2904         /* first attach mode: event */
2905         if (setup_on_probe_event) {
2906                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2907                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2908                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2909                                         ports[pi].need_setup != 0)
2910                                 setup_attached_port(pi);
2911                 return;
2912         }
2913
2914         /* second attach mode: iterator */
2915         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2916                 /* setup ports matching the devargs used for probing */
2917                 if (port_is_forwarding(pi))
2918                         continue; /* port was already attached before */
2919                 setup_attached_port(pi);
2920         }
2921 }
2922
2923 static void
2924 setup_attached_port(portid_t pi)
2925 {
2926         unsigned int socket_id;
2927         int ret;
2928
2929         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2930         /* if socket_id is invalid, set to the first available socket. */
2931         if (check_socket_id(socket_id) < 0)
2932                 socket_id = socket_ids[0];
2933         reconfig(pi, socket_id);
2934         ret = rte_eth_promiscuous_enable(pi);
2935         if (ret != 0)
2936                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2937                         pi, rte_strerror(-ret));
2938
2939         ports_ids[nb_ports++] = pi;
2940         fwd_ports_ids[nb_fwd_ports++] = pi;
2941         nb_cfg_ports = nb_fwd_ports;
2942         ports[pi].need_setup = 0;
2943         ports[pi].port_status = RTE_PORT_STOPPED;
2944
2945         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2946         printf("Done\n");
2947 }
2948
2949 static void
2950 detach_device(struct rte_device *dev)
2951 {
2952         portid_t sibling;
2953
2954         if (dev == NULL) {
2955                 printf("Device already removed\n");
2956                 return;
2957         }
2958
2959         printf("Removing a device...\n");
2960
2961         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2962                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2963                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2964                                 printf("Port %u not stopped\n", sibling);
2965                                 return;
2966                         }
2967                         port_flow_flush(sibling);
2968                 }
2969         }
2970
2971         if (rte_dev_remove(dev) < 0) {
2972                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2973                 return;
2974         }
2975         remove_invalid_ports();
2976
2977         printf("Device is detached\n");
2978         printf("Now total ports is %d\n", nb_ports);
2979         printf("Done\n");
2980         return;
2981 }
2982
2983 void
2984 detach_port_device(portid_t port_id)
2985 {
2986         if (port_id_is_invalid(port_id, ENABLED_WARN))
2987                 return;
2988
2989         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2990                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2991                         printf("Port not stopped\n");
2992                         return;
2993                 }
2994                 printf("Port was not closed\n");
2995         }
2996
2997         detach_device(rte_eth_devices[port_id].device);
2998 }
2999
3000 void
3001 detach_devargs(char *identifier)
3002 {
3003         struct rte_dev_iterator iterator;
3004         struct rte_devargs da;
3005         portid_t port_id;
3006
3007         printf("Removing a device...\n");
3008
3009         memset(&da, 0, sizeof(da));
3010         if (rte_devargs_parsef(&da, "%s", identifier)) {
3011                 printf("cannot parse identifier\n");
3012                 if (da.args)
3013                         free(da.args);
3014                 return;
3015         }
3016
3017         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3018                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3019                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3020                                 printf("Port %u not stopped\n", port_id);
3021                                 rte_eth_iterator_cleanup(&iterator);
3022                                 return;
3023                         }
3024                         port_flow_flush(port_id);
3025                 }
3026         }
3027
3028         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3029                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3030                             da.name, da.bus->name);
3031                 return;
3032         }
3033
3034         remove_invalid_ports();
3035
3036         printf("Device %s is detached\n", identifier);
3037         printf("Now total ports is %d\n", nb_ports);
3038         printf("Done\n");
3039 }
3040
3041 void
3042 pmd_test_exit(void)
3043 {
3044         portid_t pt_id;
3045         unsigned int i;
3046         int ret;
3047
3048         if (test_done == 0)
3049                 stop_packet_forwarding();
3050
3051         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3052                 if (mempools[i]) {
3053                         if (mp_alloc_type == MP_ALLOC_ANON)
3054                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3055                                                      NULL);
3056                 }
3057         }
3058         if (ports != NULL) {
3059                 no_link_check = 1;
3060                 RTE_ETH_FOREACH_DEV(pt_id) {
3061                         printf("\nStopping port %d...\n", pt_id);
3062                         fflush(stdout);
3063                         stop_port(pt_id);
3064                 }
3065                 RTE_ETH_FOREACH_DEV(pt_id) {
3066                         printf("\nShutting down port %d...\n", pt_id);
3067                         fflush(stdout);
3068                         close_port(pt_id);
3069                 }
3070         }
3071
3072         if (hot_plug) {
3073                 ret = rte_dev_event_monitor_stop();
3074                 if (ret) {
3075                         RTE_LOG(ERR, EAL,
3076                                 "fail to stop device event monitor.");
3077                         return;
3078                 }
3079
3080                 ret = rte_dev_event_callback_unregister(NULL,
3081                         dev_event_callback, NULL);
3082                 if (ret < 0) {
3083                         RTE_LOG(ERR, EAL,
3084                                 "fail to unregister device event callback.\n");
3085                         return;
3086                 }
3087
3088                 ret = rte_dev_hotplug_handle_disable();
3089                 if (ret) {
3090                         RTE_LOG(ERR, EAL,
3091                                 "fail to disable hotplug handling.\n");
3092                         return;
3093                 }
3094         }
3095         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3096                 if (mempools[i])
3097                         rte_mempool_free(mempools[i]);
3098         }
3099
3100         printf("\nBye...\n");
3101 }
3102
3103 typedef void (*cmd_func_t)(void);
3104 struct pmd_test_command {
3105         const char *cmd_name;
3106         cmd_func_t cmd_func;
3107 };
3108
3109 /* Check the link status of all ports in up to 9s, and print them finally */
3110 static void
3111 check_all_ports_link_status(uint32_t port_mask)
3112 {
3113 #define CHECK_INTERVAL 100 /* 100ms */
3114 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3115         portid_t portid;
3116         uint8_t count, all_ports_up, print_flag = 0;
3117         struct rte_eth_link link;
3118         int ret;
3119         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3120
3121         printf("Checking link statuses...\n");
3122         fflush(stdout);
3123         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3124                 all_ports_up = 1;
3125                 RTE_ETH_FOREACH_DEV(portid) {
3126                         if ((port_mask & (1 << portid)) == 0)
3127                                 continue;
3128                         memset(&link, 0, sizeof(link));
3129                         ret = rte_eth_link_get_nowait(portid, &link);
3130                         if (ret < 0) {
3131                                 all_ports_up = 0;
3132                                 if (print_flag == 1)
3133                                         printf("Port %u link get failed: %s\n",
3134                                                 portid, rte_strerror(-ret));
3135                                 continue;
3136                         }
3137                         /* print link status if flag set */
3138                         if (print_flag == 1) {
3139                                 rte_eth_link_to_str(link_status,
3140                                         sizeof(link_status), &link);
3141                                 printf("Port %d %s\n", portid, link_status);
3142                                 continue;
3143                         }
3144                         /* clear all_ports_up flag if any link down */
3145                         if (link.link_status == ETH_LINK_DOWN) {
3146                                 all_ports_up = 0;
3147                                 break;
3148                         }
3149                 }
3150                 /* after finally printing all link status, get out */
3151                 if (print_flag == 1)
3152                         break;
3153
3154                 if (all_ports_up == 0) {
3155                         fflush(stdout);
3156                         rte_delay_ms(CHECK_INTERVAL);
3157                 }
3158
3159                 /* set the print_flag if all ports up or timeout */
3160                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3161                         print_flag = 1;
3162                 }
3163
3164                 if (lsc_interrupt)
3165                         break;
3166         }
3167 }
3168
3169 static void
3170 rmv_port_callback(void *arg)
3171 {
3172         int need_to_start = 0;
3173         int org_no_link_check = no_link_check;
3174         portid_t port_id = (intptr_t)arg;
3175         struct rte_device *dev;
3176
3177         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3178
3179         if (!test_done && port_is_forwarding(port_id)) {
3180                 need_to_start = 1;
3181                 stop_packet_forwarding();
3182         }
3183         no_link_check = 1;
3184         stop_port(port_id);
3185         no_link_check = org_no_link_check;
3186
3187         /* Save rte_device pointer before closing ethdev port */
3188         dev = rte_eth_devices[port_id].device;
3189         close_port(port_id);
3190         detach_device(dev); /* might be already removed or have more ports */
3191
3192         if (need_to_start)
3193                 start_packet_forwarding(0);
3194 }
3195
3196 /* This function is used by the interrupt thread */
3197 static int
3198 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3199                   void *ret_param)
3200 {
3201         RTE_SET_USED(param);
3202         RTE_SET_USED(ret_param);
3203
3204         if (type >= RTE_ETH_EVENT_MAX) {
3205                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3206                         port_id, __func__, type);
3207                 fflush(stderr);
3208         } else if (event_print_mask & (UINT32_C(1) << type)) {
3209                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3210                         eth_event_desc[type]);
3211                 fflush(stdout);
3212         }
3213
3214         switch (type) {
3215         case RTE_ETH_EVENT_NEW:
3216                 ports[port_id].need_setup = 1;
3217                 ports[port_id].port_status = RTE_PORT_HANDLING;
3218                 break;
3219         case RTE_ETH_EVENT_INTR_RMV:
3220                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3221                         break;
3222                 if (rte_eal_alarm_set(100000,
3223                                 rmv_port_callback, (void *)(intptr_t)port_id))
3224                         fprintf(stderr, "Could not set up deferred device removal\n");
3225                 break;
3226         case RTE_ETH_EVENT_DESTROY:
3227                 ports[port_id].port_status = RTE_PORT_CLOSED;
3228                 printf("Port %u is closed\n", port_id);
3229                 break;
3230         default:
3231                 break;
3232         }
3233         return 0;
3234 }
3235
3236 static int
3237 register_eth_event_callback(void)
3238 {
3239         int ret;
3240         enum rte_eth_event_type event;
3241
3242         for (event = RTE_ETH_EVENT_UNKNOWN;
3243                         event < RTE_ETH_EVENT_MAX; event++) {
3244                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3245                                 event,
3246                                 eth_event_callback,
3247                                 NULL);
3248                 if (ret != 0) {
3249                         TESTPMD_LOG(ERR, "Failed to register callback for "
3250                                         "%s event\n", eth_event_desc[event]);
3251                         return -1;
3252                 }
3253         }
3254
3255         return 0;
3256 }
3257
3258 /* This function is used by the interrupt thread */
3259 static void
3260 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3261                              __rte_unused void *arg)
3262 {
3263         uint16_t port_id;
3264         int ret;
3265
3266         if (type >= RTE_DEV_EVENT_MAX) {
3267                 fprintf(stderr, "%s called upon invalid event %d\n",
3268                         __func__, type);
3269                 fflush(stderr);
3270         }
3271
3272         switch (type) {
3273         case RTE_DEV_EVENT_REMOVE:
3274                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3275                         device_name);
3276                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3277                 if (ret) {
3278                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3279                                 device_name);
3280                         return;
3281                 }
3282                 /*
3283                  * Because the user's callback is invoked in eal interrupt
3284                  * callback, the interrupt callback need to be finished before
3285                  * it can be unregistered when detaching device. So finish
3286                  * callback soon and use a deferred removal to detach device
3287                  * is need. It is a workaround, once the device detaching be
3288                  * moved into the eal in the future, the deferred removal could
3289                  * be deleted.
3290                  */
3291                 if (rte_eal_alarm_set(100000,
3292                                 rmv_port_callback, (void *)(intptr_t)port_id))
3293                         RTE_LOG(ERR, EAL,
3294                                 "Could not set up deferred device removal\n");
3295                 break;
3296         case RTE_DEV_EVENT_ADD:
3297                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3298                         device_name);
3299                 /* TODO: After finish kernel driver binding,
3300                  * begin to attach port.
3301                  */
3302                 break;
3303         default:
3304                 break;
3305         }
3306 }
3307
3308 static int
3309 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3310 {
3311         uint16_t i;
3312         int diag;
3313         uint8_t mapping_found = 0;
3314
3315         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3316                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3317                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3318                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3319                                         tx_queue_stats_mappings[i].queue_id,
3320                                         tx_queue_stats_mappings[i].stats_counter_id);
3321                         if (diag != 0)
3322                                 return diag;
3323                         mapping_found = 1;
3324                 }
3325         }
3326         if (mapping_found)
3327                 port->tx_queue_stats_mapping_enabled = 1;
3328         return 0;
3329 }
3330
3331 static int
3332 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3333 {
3334         uint16_t i;
3335         int diag;
3336         uint8_t mapping_found = 0;
3337
3338         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3339                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3340                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3341                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3342                                         rx_queue_stats_mappings[i].queue_id,
3343                                         rx_queue_stats_mappings[i].stats_counter_id);
3344                         if (diag != 0)
3345                                 return diag;
3346                         mapping_found = 1;
3347                 }
3348         }
3349         if (mapping_found)
3350                 port->rx_queue_stats_mapping_enabled = 1;
3351         return 0;
3352 }
3353
3354 static void
3355 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3356 {
3357         int diag = 0;
3358
3359         diag = set_tx_queue_stats_mapping_registers(pi, port);
3360         if (diag != 0) {
3361                 if (diag == -ENOTSUP) {
3362                         port->tx_queue_stats_mapping_enabled = 0;
3363                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3364                 }
3365                 else
3366                         rte_exit(EXIT_FAILURE,
3367                                         "set_tx_queue_stats_mapping_registers "
3368                                         "failed for port id=%d diag=%d\n",
3369                                         pi, diag);
3370         }
3371
3372         diag = set_rx_queue_stats_mapping_registers(pi, port);
3373         if (diag != 0) {
3374                 if (diag == -ENOTSUP) {
3375                         port->rx_queue_stats_mapping_enabled = 0;
3376                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3377                 }
3378                 else
3379                         rte_exit(EXIT_FAILURE,
3380                                         "set_rx_queue_stats_mapping_registers "
3381                                         "failed for port id=%d diag=%d\n",
3382                                         pi, diag);
3383         }
3384 }
3385
3386 static void
3387 rxtx_port_config(struct rte_port *port)
3388 {
3389         uint16_t qid;
3390         uint64_t offloads;
3391
3392         for (qid = 0; qid < nb_rxq; qid++) {
3393                 offloads = port->rx_conf[qid].offloads;
3394                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3395                 if (offloads != 0)
3396                         port->rx_conf[qid].offloads = offloads;
3397
3398                 /* Check if any Rx parameters have been passed */
3399                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3400                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3401
3402                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3403                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3404
3405                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3406                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3407
3408                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3409                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3410
3411                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3412                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3413
3414                 port->nb_rx_desc[qid] = nb_rxd;
3415         }
3416
3417         for (qid = 0; qid < nb_txq; qid++) {
3418                 offloads = port->tx_conf[qid].offloads;
3419                 port->tx_conf[qid] = port->dev_info.default_txconf;
3420                 if (offloads != 0)
3421                         port->tx_conf[qid].offloads = offloads;
3422
3423                 /* Check if any Tx parameters have been passed */
3424                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3425                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3426
3427                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3428                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3429
3430                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3431                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3432
3433                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3434                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3435
3436                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3437                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3438
3439                 port->nb_tx_desc[qid] = nb_txd;
3440         }
3441 }
3442
3443 void
3444 init_port_config(void)
3445 {
3446         portid_t pid;
3447         struct rte_port *port;
3448         int ret;
3449
3450         RTE_ETH_FOREACH_DEV(pid) {
3451                 port = &ports[pid];
3452                 port->dev_conf.fdir_conf = fdir_conf;
3453
3454                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3455                 if (ret != 0)
3456                         return;
3457
3458                 if (nb_rxq > 1) {
3459                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3460                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3461                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3462                 } else {
3463                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3464                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3465                 }
3466
3467                 if (port->dcb_flag == 0) {
3468                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3469                                 port->dev_conf.rxmode.mq_mode =
3470                                         (enum rte_eth_rx_mq_mode)
3471                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3472                         else
3473                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3474                 }
3475
3476                 rxtx_port_config(port);
3477
3478                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3479                 if (ret != 0)
3480                         return;
3481
3482                 map_port_queue_stats_mapping_registers(pid, port);
3483 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3484                 rte_pmd_ixgbe_bypass_init(pid);
3485 #endif
3486
3487                 if (lsc_interrupt &&
3488                     (rte_eth_devices[pid].data->dev_flags &
3489                      RTE_ETH_DEV_INTR_LSC))
3490                         port->dev_conf.intr_conf.lsc = 1;
3491                 if (rmv_interrupt &&
3492                     (rte_eth_devices[pid].data->dev_flags &
3493                      RTE_ETH_DEV_INTR_RMV))
3494                         port->dev_conf.intr_conf.rmv = 1;
3495         }
3496 }
3497
3498 void set_port_slave_flag(portid_t slave_pid)
3499 {
3500         struct rte_port *port;
3501
3502         port = &ports[slave_pid];
3503         port->slave_flag = 1;
3504 }
3505
3506 void clear_port_slave_flag(portid_t slave_pid)
3507 {
3508         struct rte_port *port;
3509
3510         port = &ports[slave_pid];
3511         port->slave_flag = 0;
3512 }
3513
3514 uint8_t port_is_bonding_slave(portid_t slave_pid)
3515 {
3516         struct rte_port *port;
3517
3518         port = &ports[slave_pid];
3519         if ((rte_eth_devices[slave_pid].data->dev_flags &
3520             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3521                 return 1;
3522         return 0;
3523 }
3524
3525 const uint16_t vlan_tags[] = {
3526                 0,  1,  2,  3,  4,  5,  6,  7,
3527                 8,  9, 10, 11,  12, 13, 14, 15,
3528                 16, 17, 18, 19, 20, 21, 22, 23,
3529                 24, 25, 26, 27, 28, 29, 30, 31
3530 };
3531
3532 static  int
3533 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3534                  enum dcb_mode_enable dcb_mode,
3535                  enum rte_eth_nb_tcs num_tcs,
3536                  uint8_t pfc_en)
3537 {
3538         uint8_t i;
3539         int32_t rc;
3540         struct rte_eth_rss_conf rss_conf;
3541
3542         /*
3543          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3544          * given above, and the number of traffic classes available for use.
3545          */
3546         if (dcb_mode == DCB_VT_ENABLED) {
3547                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3548                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3549                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3550                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3551
3552                 /* VMDQ+DCB RX and TX configurations */
3553                 vmdq_rx_conf->enable_default_pool = 0;
3554                 vmdq_rx_conf->default_pool = 0;
3555                 vmdq_rx_conf->nb_queue_pools =
3556                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3557                 vmdq_tx_conf->nb_queue_pools =
3558                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3559
3560                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3561                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3562                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3563                         vmdq_rx_conf->pool_map[i].pools =
3564                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3565                 }
3566                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3567                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3568                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3569                 }
3570
3571                 /* set DCB mode of RX and TX of multiple queues */
3572                 eth_conf->rxmode.mq_mode =
3573                                 (enum rte_eth_rx_mq_mode)
3574                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3575                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3576         } else {
3577                 struct rte_eth_dcb_rx_conf *rx_conf =
3578                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3579                 struct rte_eth_dcb_tx_conf *tx_conf =
3580                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3581
3582                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3583
3584                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3585                 if (rc != 0)
3586                         return rc;
3587
3588                 rx_conf->nb_tcs = num_tcs;
3589                 tx_conf->nb_tcs = num_tcs;
3590
3591                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3592                         rx_conf->dcb_tc[i] = i % num_tcs;
3593                         tx_conf->dcb_tc[i] = i % num_tcs;
3594                 }
3595
3596                 eth_conf->rxmode.mq_mode =
3597                                 (enum rte_eth_rx_mq_mode)
3598                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3599                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3600                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3601         }
3602
3603         if (pfc_en)
3604                 eth_conf->dcb_capability_en =
3605                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3606         else
3607                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3608
3609         return 0;
3610 }
3611
3612 int
3613 init_port_dcb_config(portid_t pid,
3614                      enum dcb_mode_enable dcb_mode,
3615                      enum rte_eth_nb_tcs num_tcs,
3616                      uint8_t pfc_en)
3617 {
3618         struct rte_eth_conf port_conf;
3619         struct rte_port *rte_port;
3620         int retval;
3621         uint16_t i;
3622
3623         rte_port = &ports[pid];
3624
3625         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3626         /* Enter DCB configuration status */
3627         dcb_config = 1;
3628
3629         port_conf.rxmode = rte_port->dev_conf.rxmode;
3630         port_conf.txmode = rte_port->dev_conf.txmode;
3631
3632         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3633         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3634         if (retval < 0)
3635                 return retval;
3636         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3637
3638         /* re-configure the device . */
3639         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3640         if (retval < 0)
3641                 return retval;
3642
3643         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3644         if (retval != 0)
3645                 return retval;
3646
3647         /* If dev_info.vmdq_pool_base is greater than 0,
3648          * the queue id of vmdq pools is started after pf queues.
3649          */
3650         if (dcb_mode == DCB_VT_ENABLED &&
3651             rte_port->dev_info.vmdq_pool_base > 0) {
3652                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3653                         " for port %d.", pid);
3654                 return -1;
3655         }
3656
3657         /* Assume the ports in testpmd have the same dcb capability
3658          * and has the same number of rxq and txq in dcb mode
3659          */
3660         if (dcb_mode == DCB_VT_ENABLED) {
3661                 if (rte_port->dev_info.max_vfs > 0) {
3662                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3663                         nb_txq = rte_port->dev_info.nb_tx_queues;
3664                 } else {
3665                         nb_rxq = rte_port->dev_info.max_rx_queues;
3666                         nb_txq = rte_port->dev_info.max_tx_queues;
3667                 }
3668         } else {
3669                 /*if vt is disabled, use all pf queues */
3670                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3671                         nb_rxq = rte_port->dev_info.max_rx_queues;
3672                         nb_txq = rte_port->dev_info.max_tx_queues;
3673                 } else {
3674                         nb_rxq = (queueid_t)num_tcs;
3675                         nb_txq = (queueid_t)num_tcs;
3676
3677                 }
3678         }
3679         rx_free_thresh = 64;
3680
3681         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3682
3683         rxtx_port_config(rte_port);
3684         /* VLAN filter */
3685         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3686         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3687                 rx_vft_set(pid, vlan_tags[i], 1);
3688
3689         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3690         if (retval != 0)
3691                 return retval;
3692
3693         map_port_queue_stats_mapping_registers(pid, rte_port);
3694
3695         rte_port->dcb_flag = 1;
3696
3697         return 0;
3698 }
3699
3700 static void
3701 init_port(void)
3702 {
3703         int i;
3704
3705         /* Configuration of Ethernet ports. */
3706         ports = rte_zmalloc("testpmd: ports",
3707                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3708                             RTE_CACHE_LINE_SIZE);
3709         if (ports == NULL) {
3710                 rte_exit(EXIT_FAILURE,
3711                                 "rte_zmalloc(%d struct rte_port) failed\n",
3712                                 RTE_MAX_ETHPORTS);
3713         }
3714         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3715                 LIST_INIT(&ports[i].flow_tunnel_list);
3716         /* Initialize ports NUMA structures */
3717         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3718         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3719         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3720 }
3721
3722 static void
3723 force_quit(void)
3724 {
3725         pmd_test_exit();
3726         prompt_exit();
3727 }
3728
3729 static void
3730 print_stats(void)
3731 {
3732         uint8_t i;
3733         const char clr[] = { 27, '[', '2', 'J', '\0' };
3734         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3735
3736         /* Clear screen and move to top left */
3737         printf("%s%s", clr, top_left);
3738
3739         printf("\nPort statistics ====================================");
3740         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3741                 nic_stats_display(fwd_ports_ids[i]);
3742
3743         fflush(stdout);
3744 }
3745
3746 static void
3747 signal_handler(int signum)
3748 {
3749         if (signum == SIGINT || signum == SIGTERM) {
3750                 printf("\nSignal %d received, preparing to exit...\n",
3751                                 signum);
3752 #ifdef RTE_LIBRTE_PDUMP
3753                 /* uninitialize packet capture framework */
3754                 rte_pdump_uninit();
3755 #endif
3756 #ifdef RTE_LIBRTE_LATENCY_STATS
3757                 if (latencystats_enabled != 0)
3758                         rte_latencystats_uninit();
3759 #endif
3760                 force_quit();
3761                 /* Set flag to indicate the force termination. */
3762                 f_quit = 1;
3763                 /* exit with the expected status */
3764                 signal(signum, SIG_DFL);
3765                 kill(getpid(), signum);
3766         }
3767 }
3768
3769 int
3770 main(int argc, char** argv)
3771 {
3772         int diag;
3773         portid_t port_id;
3774         uint16_t count;
3775         int ret;
3776
3777         signal(SIGINT, signal_handler);
3778         signal(SIGTERM, signal_handler);
3779
3780         testpmd_logtype = rte_log_register("testpmd");
3781         if (testpmd_logtype < 0)
3782                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3783         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3784
3785         diag = rte_eal_init(argc, argv);
3786         if (diag < 0)
3787                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3788                          rte_strerror(rte_errno));
3789
3790         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3791                 rte_exit(EXIT_FAILURE,
3792                          "Secondary process type not supported.\n");
3793
3794         ret = register_eth_event_callback();
3795         if (ret != 0)
3796                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3797
3798 #ifdef RTE_LIBRTE_PDUMP
3799         /* initialize packet capture framework */
3800         rte_pdump_init();
3801 #endif
3802
3803         count = 0;
3804         RTE_ETH_FOREACH_DEV(port_id) {
3805                 ports_ids[count] = port_id;
3806                 count++;
3807         }
3808         nb_ports = (portid_t) count;
3809         if (nb_ports == 0)
3810                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3811
3812         /* allocate port structures, and init them */
3813         init_port();
3814
3815         set_def_fwd_config();
3816         if (nb_lcores == 0)
3817                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3818                          "Check the core mask argument\n");
3819
3820         /* Bitrate/latency stats disabled by default */
3821 #ifdef RTE_LIBRTE_BITRATESTATS
3822         bitrate_enabled = 0;
3823 #endif
3824 #ifdef RTE_LIBRTE_LATENCY_STATS
3825         latencystats_enabled = 0;
3826 #endif
3827
3828         /* on FreeBSD, mlockall() is disabled by default */
3829 #ifdef RTE_EXEC_ENV_FREEBSD
3830         do_mlockall = 0;
3831 #else
3832         do_mlockall = 1;
3833 #endif
3834
3835         argc -= diag;
3836         argv += diag;
3837         if (argc > 1)
3838                 launch_args_parse(argc, argv);
3839
3840         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3841                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3842                         strerror(errno));
3843         }
3844
3845         if (tx_first && interactive)
3846                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3847                                 "interactive mode.\n");
3848
3849         if (tx_first && lsc_interrupt) {
3850                 printf("Warning: lsc_interrupt needs to be off when "
3851                                 " using tx_first. Disabling.\n");
3852                 lsc_interrupt = 0;
3853         }
3854
3855         if (!nb_rxq && !nb_txq)
3856                 printf("Warning: Either rx or tx queues should be non-zero\n");
3857
3858         if (nb_rxq > 1 && nb_rxq > nb_txq)
3859                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3860                        "but nb_txq=%d will prevent to fully test it.\n",
3861                        nb_rxq, nb_txq);
3862
3863         init_config();
3864
3865         if (hot_plug) {
3866                 ret = rte_dev_hotplug_handle_enable();
3867                 if (ret) {
3868                         RTE_LOG(ERR, EAL,
3869                                 "fail to enable hotplug handling.");
3870                         return -1;
3871                 }
3872
3873                 ret = rte_dev_event_monitor_start();
3874                 if (ret) {
3875                         RTE_LOG(ERR, EAL,
3876                                 "fail to start device event monitoring.");
3877                         return -1;
3878                 }
3879
3880                 ret = rte_dev_event_callback_register(NULL,
3881                         dev_event_callback, NULL);
3882                 if (ret) {
3883                         RTE_LOG(ERR, EAL,
3884                                 "fail  to register device event callback\n");
3885                         return -1;
3886                 }
3887         }
3888
3889         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3890                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3891
3892         /* set all ports to promiscuous mode by default */
3893         RTE_ETH_FOREACH_DEV(port_id) {
3894                 ret = rte_eth_promiscuous_enable(port_id);
3895                 if (ret != 0)
3896                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3897                                 port_id, rte_strerror(-ret));
3898         }
3899
3900         /* Init metrics library */
3901         rte_metrics_init(rte_socket_id());
3902
3903 #ifdef RTE_LIBRTE_LATENCY_STATS
3904         if (latencystats_enabled != 0) {
3905                 int ret = rte_latencystats_init(1, NULL);
3906                 if (ret)
3907                         printf("Warning: latencystats init()"
3908                                 " returned error %d\n", ret);
3909                 printf("Latencystats running on lcore %d\n",
3910                         latencystats_lcore_id);
3911         }
3912 #endif
3913
3914         /* Setup bitrate stats */
3915 #ifdef RTE_LIBRTE_BITRATESTATS
3916         if (bitrate_enabled != 0) {
3917                 bitrate_data = rte_stats_bitrate_create();
3918                 if (bitrate_data == NULL)
3919                         rte_exit(EXIT_FAILURE,
3920                                 "Could not allocate bitrate data.\n");
3921                 rte_stats_bitrate_reg(bitrate_data);
3922         }
3923 #endif
3924
3925 #ifdef RTE_LIBRTE_CMDLINE
3926         if (strlen(cmdline_filename) != 0)
3927                 cmdline_read_from_file(cmdline_filename);
3928
3929         if (interactive == 1) {
3930                 if (auto_start) {
3931                         printf("Start automatic packet forwarding\n");
3932                         start_packet_forwarding(0);
3933                 }
3934                 prompt();
3935                 pmd_test_exit();
3936         } else
3937 #endif
3938         {
3939                 char c;
3940                 int rc;
3941
3942                 f_quit = 0;
3943
3944                 printf("No commandline core given, start packet forwarding\n");
3945                 start_packet_forwarding(tx_first);
3946                 if (stats_period != 0) {
3947                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3948                         uint64_t timer_period;
3949
3950                         /* Convert to number of cycles */
3951                         timer_period = stats_period * rte_get_timer_hz();
3952
3953                         while (f_quit == 0) {
3954                                 cur_time = rte_get_timer_cycles();
3955                                 diff_time += cur_time - prev_time;
3956
3957                                 if (diff_time >= timer_period) {
3958                                         print_stats();
3959                                         /* Reset the timer */
3960                                         diff_time = 0;
3961                                 }
3962                                 /* Sleep to avoid unnecessary checks */
3963                                 prev_time = cur_time;
3964                                 sleep(1);
3965                         }
3966                 }
3967
3968                 printf("Press enter to exit\n");
3969                 rc = read(0, &c, 1);
3970                 pmd_test_exit();
3971                 if (rc < 0)
3972                         return 1;
3973         }
3974
3975         ret = rte_eal_cleanup();
3976         if (ret != 0)
3977                 rte_exit(EXIT_FAILURE,
3978                          "EAL cleanup failed: %s\n", strerror(-ret));
3979
3980         return EXIT_SUCCESS;
3981 }