app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATESTATS
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 200         DEFAULT_MBUF_DATA_SIZE
 201 }; /**< Mbuf data space size. */
 202 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 203                                       * specified on command-line. */
 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 205
 206 /*
 207  * In container, it cannot terminate the process which running with 'stats-period'
 208  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 209  */
 210 uint8_t f_quit;
 211
 212 /*
 213  * Configuration of packet segments used by the "txonly" processing engine.
 214  */
 215 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 216 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 217         TXONLY_DEF_PACKET_LEN,
 218 };
 219 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 220
 221 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 222 /**< Split policy for packets to TX. */
 223
 224 uint8_t txonly_multi_flow;
 225 /**< Whether multiple flows are generated in TXONLY mode. */
 226
 227 uint32_t tx_pkt_times_inter;
 228 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 229
 230 uint32_t tx_pkt_times_intra;
 231 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 232
 233 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 234 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 235
 236 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 237 uint8_t dcb_config = 0;
 238
 239 /* Whether the dcb is in testing status */
 240 uint8_t dcb_test = 0;
 241
 242 /*
 243  * Configurable number of RX/TX queues.
 244  */
 245 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 246 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 247 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 248
 249 /*
 250  * Configurable number of RX/TX ring descriptors.
 251  * Defaults are supplied by drivers via ethdev.
 252  */
 253 #define RTE_TEST_RX_DESC_DEFAULT 0
 254 #define RTE_TEST_TX_DESC_DEFAULT 0
 255 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 256 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 257
 258 #define RTE_PMD_PARAM_UNSET -1
 259 /*
 260  * Configurable values of RX and TX ring threshold registers.
 261  */
 262
 263 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 264 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 265 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 266
 267 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 268 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 269 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 270
 271 /*
 272  * Configurable value of RX free threshold.
 273  */
 274 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 275
 276 /*
 277  * Configurable value of RX drop enable.
 278  */
 279 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 280
 281 /*
 282  * Configurable value of TX free threshold.
 283  */
 284 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 285
 286 /*
 287  * Configurable value of TX RS bit threshold.
 288  */
 289 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 290
 291 /*
 292  * Configurable value of buffered packets before sending.
 293  */
 294 uint16_t noisy_tx_sw_bufsz;
 295
 296 /*
 297  * Configurable value of packet buffer timeout.
 298  */
 299 uint16_t noisy_tx_sw_buf_flush_time;
 300
 301 /*
 302  * Configurable value for size of VNF internal memory area
 303  * used for simulating noisy neighbour behaviour
 304  */
 305 uint64_t noisy_lkup_mem_sz;
 306
 307 /*
 308  * Configurable value of number of random writes done in
 309  * VNF simulation memory area.
 310  */
 311 uint64_t noisy_lkup_num_writes;
 312
 313 /*
 314  * Configurable value of number of random reads done in
 315  * VNF simulation memory area.
 316  */
 317 uint64_t noisy_lkup_num_reads;
 318
 319 /*
 320  * Configurable value of number of random reads/writes done in
 321  * VNF simulation memory area.
 322  */
 323 uint64_t noisy_lkup_num_reads_writes;
 324
 325 /*
 326  * Receive Side Scaling (RSS) configuration.
 327  */
 328 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 329
 330 /*
 331  * Port topology configuration
 332  */
 333 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 334
 335 /*
 336  * Avoids to flush all the RX streams before starts forwarding.
 337  */
 338 uint8_t no_flush_rx = 0; /* flush by default */
 339
 340 /*
 341  * Flow API isolated mode.
 342  */
 343 uint8_t flow_isolate_all;
 344
 345 /*
 346  * Avoids to check link status when starting/stopping a port.
 347  */
 348 uint8_t no_link_check = 0; /* check by default */
 349
 350 /*
 351  * Don't automatically start all ports in interactive mode.
 352  */
 353 uint8_t no_device_start = 0;
 354
 355 /*
 356  * Enable link status change notification
 357  */
 358 uint8_t lsc_interrupt = 1; /* enabled by default */
 359
 360 /*
 361  * Enable device removal notification.
 362  */
 363 uint8_t rmv_interrupt = 1; /* enabled by default */
 364
 365 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 366
 367 /* After attach, port setup is called on event or by iterator */
 368 bool setup_on_probe_event = true;
 369
 370 /* Clear ptypes on port initialization. */
 371 uint8_t clear_ptypes = true;
 372
 373 /* Hairpin ports configuration mode. */
 374 uint16_t hairpin_mode;
 375
 376 /* Pretty printing of ethdev events */
 377 static const char * const eth_event_desc[] = {
 378         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 379         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 380         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 381         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 382         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 383         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 384         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 385         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 386         [RTE_ETH_EVENT_NEW] = "device probed",
 387         [RTE_ETH_EVENT_DESTROY] = "device released",
 388         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 389         [RTE_ETH_EVENT_MAX] = NULL,
 390 };
 391
 392 /*
 393  * Display or mask ether events
 394  * Default to all events except VF_MBOX
 395  */
 396 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 397                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 398                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 399                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 400                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 401                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 402                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 403                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 404 /*
 405  * Decide if all memory are locked for performance.
 406  */
 407 int do_mlockall = 0;
 408
 409 /*
 410  * NIC bypass mode configuration options.
 411  */
 412
 413 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 414 /* The NIC bypass watchdog timeout. */
 415 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 416 #endif
 417
 418
 419 #ifdef RTE_LIBRTE_LATENCY_STATS
 420
 421 /*
 422  * Set when latency stats is enabled in the commandline
 423  */
 424 uint8_t latencystats_enabled;
 425
 426 /*
 427  * Lcore ID to serive latency statistics.
 428  */
 429 lcoreid_t latencystats_lcore_id = -1;
 430
 431 #endif
 432
 433 /*
 434  * Ethernet device configuration.
 435  */
 436 struct rte_eth_rxmode rx_mode = {
 437         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 438                 /**< Default maximum frame length. */
 439 };
 440
 441 struct rte_eth_txmode tx_mode = {
 442         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 443 };
 444
 445 struct rte_fdir_conf fdir_conf = {
 446         .mode = RTE_FDIR_MODE_NONE,
 447         .pballoc = RTE_FDIR_PBALLOC_64K,
 448         .status = RTE_FDIR_REPORT_STATUS,
 449         .mask = {
 450                 .vlan_tci_mask = 0xFFEF,
 451                 .ipv4_mask     = {
 452                         .src_ip = 0xFFFFFFFF,
 453                         .dst_ip = 0xFFFFFFFF,
 454                 },
 455                 .ipv6_mask     = {
 456                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 457                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 458                 },
 459                 .src_port_mask = 0xFFFF,
 460                 .dst_port_mask = 0xFFFF,
 461                 .mac_addr_byte_mask = 0xFF,
 462                 .tunnel_type_mask = 1,
 463                 .tunnel_id_mask = 0xFFFFFFFF,
 464         },
 465         .drop_queue = 127,
 466 };
 467
 468 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 469
 470 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 471 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 472
 473 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 474 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 475
 476 uint16_t nb_tx_queue_stats_mappings = 0;
 477 uint16_t nb_rx_queue_stats_mappings = 0;
 478
 479 /*
 480  * Display zero values by default for xstats
 481  */
 482 uint8_t xstats_hide_zero;
 483
 484 /*
 485  * Measure of CPU cycles disabled by default
 486  */
 487 uint8_t record_core_cycles;
 488
 489 /*
 490  * Display of RX and TX bursts disabled by default
 491  */
 492 uint8_t record_burst_stats;
 493
 494 unsigned int num_sockets = 0;
 495 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 496
 497 #ifdef RTE_LIBRTE_BITRATESTATS
 498 /* Bitrate statistics */
 499 struct rte_stats_bitrates *bitrate_data;
 500 lcoreid_t bitrate_lcore_id;
 501 uint8_t bitrate_enabled;
 502 #endif
 503
 504 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 505 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 506
 507 /*
 508  * hexadecimal bitmask of RX mq mode can be enabled.
 509  */
 510 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 511
 512 /* Forward function declarations */
 513 static void setup_attached_port(portid_t pi);
 514 static void map_port_queue_stats_mapping_registers(portid_t pi,
 515                                                    struct rte_port *port);
 516 static void check_all_ports_link_status(uint32_t port_mask);
 517 static int eth_event_callback(portid_t port_id,
 518                               enum rte_eth_event_type type,
 519                               void *param, void *ret_param);
 520 static void dev_event_callback(const char *device_name,
 521                                 enum rte_dev_event_type type,
 522                                 void *param);
 523
 524 /*
 525  * Check if all the ports are started.
 526  * If yes, return positive value. If not, return zero.
 527  */
 528 static int all_ports_started(void);
 529
 530 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 531 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 532
 533 /* Holds the registered mbuf dynamic flags names. */
 534 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 535
 536 /*
 537  * Helper function to check if socket is already discovered.
 538  * If yes, return positive value. If not, return zero.
 539  */
 540 int
 541 new_socket_id(unsigned int socket_id)
 542 {
 543         unsigned int i;
 544
 545         for (i = 0; i < num_sockets; i++) {
 546                 if (socket_ids[i] == socket_id)
 547                         return 0;
 548         }
 549         return 1;
 550 }
 551
 552 /*
 553  * Setup default configuration.
 554  */
 555 static void
 556 set_default_fwd_lcores_config(void)
 557 {
 558         unsigned int i;
 559         unsigned int nb_lc;
 560         unsigned int sock_num;
 561
 562         nb_lc = 0;
 563         for (i = 0; i < RTE_MAX_LCORE; i++) {
 564                 if (!rte_lcore_is_enabled(i))
 565                         continue;
 566                 sock_num = rte_lcore_to_socket_id(i);
 567                 if (new_socket_id(sock_num)) {
 568                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 569                                 rte_exit(EXIT_FAILURE,
 570                                          "Total sockets greater than %u\n",
 571                                          RTE_MAX_NUMA_NODES);
 572                         }
 573                         socket_ids[num_sockets++] = sock_num;
 574                 }
 575                 if (i == rte_get_master_lcore())
 576                         continue;
 577                 fwd_lcores_cpuids[nb_lc++] = i;
 578         }
 579         nb_lcores = (lcoreid_t) nb_lc;
 580         nb_cfg_lcores = nb_lcores;
 581         nb_fwd_lcores = 1;
 582 }
 583
 584 static void
 585 set_def_peer_eth_addrs(void)
 586 {
 587         portid_t i;
 588
 589         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 590                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 591                 peer_eth_addrs[i].addr_bytes[5] = i;
 592         }
 593 }
 594
 595 static void
 596 set_default_fwd_ports_config(void)
 597 {
 598         portid_t pt_id;
 599         int i = 0;
 600
 601         RTE_ETH_FOREACH_DEV(pt_id) {
 602                 fwd_ports_ids[i++] = pt_id;
 603
 604                 /* Update sockets info according to the attached device */
 605                 int socket_id = rte_eth_dev_socket_id(pt_id);
 606                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 607                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 608                                 rte_exit(EXIT_FAILURE,
 609                                          "Total sockets greater than %u\n",
 610                                          RTE_MAX_NUMA_NODES);
 611                         }
 612                         socket_ids[num_sockets++] = socket_id;
 613                 }
 614         }
 615
 616         nb_cfg_ports = nb_ports;
 617         nb_fwd_ports = nb_ports;
 618 }
 619
 620 void
 621 set_def_fwd_config(void)
 622 {
 623         set_default_fwd_lcores_config();
 624         set_def_peer_eth_addrs();
 625         set_default_fwd_ports_config();
 626 }
 627
 628 /* extremely pessimistic estimation of memory required to create a mempool */
 629 static int
 630 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 631 {
 632         unsigned int n_pages, mbuf_per_pg, leftover;
 633         uint64_t total_mem, mbuf_mem, obj_sz;
 634
 635         /* there is no good way to predict how much space the mempool will
 636          * occupy because it will allocate chunks on the fly, and some of those
 637          * will come from default DPDK memory while some will come from our
 638          * external memory, so just assume 128MB will be enough for everyone.
 639          */
 640         uint64_t hdr_mem = 128 << 20;
 641
 642         /* account for possible non-contiguousness */
 643         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 644         if (obj_sz > pgsz) {
 645                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 646                 return -1;
 647         }
 648
 649         mbuf_per_pg = pgsz / obj_sz;
 650         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 651         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 652
 653         mbuf_mem = n_pages * pgsz;
 654
 655         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 656
 657         if (total_mem > SIZE_MAX) {
 658                 TESTPMD_LOG(ERR, "Memory size too big\n");
 659                 return -1;
 660         }
 661         *out = (size_t)total_mem;
 662
 663         return 0;
 664 }
 665
 666 static int
 667 pagesz_flags(uint64_t page_sz)
 668 {
 669         /* as per mmap() manpage, all page sizes are log2 of page size
 670          * shifted by MAP_HUGE_SHIFT
 671          */
 672         int log2 = rte_log2_u64(page_sz);
 673
 674         return (log2 << HUGE_SHIFT);
 675 }
 676
 677 static void *
 678 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 679 {
 680         void *addr;
 681         int flags;
 682
 683         /* allocate anonymous hugepages */
 684         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 685         if (huge)
 686                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 687
 688         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 689         if (addr == MAP_FAILED)
 690                 return NULL;
 691
 692         return addr;
 693 }
 694
 695 struct extmem_param {
 696         void *addr;
 697         size_t len;
 698         size_t pgsz;
 699         rte_iova_t *iova_table;
 700         unsigned int iova_table_len;
 701 };
 702
 703 static int
 704 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 705                 bool huge)
 706 {
 707         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 708                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 709         unsigned int cur_page, n_pages, pgsz_idx;
 710         size_t mem_sz, cur_pgsz;
 711         rte_iova_t *iovas = NULL;
 712         void *addr;
 713         int ret;
 714
 715         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 716                 /* skip anything that is too big */
 717                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 718                         continue;
 719
 720                 cur_pgsz = pgsizes[pgsz_idx];
 721
 722                 /* if we were told not to allocate hugepages, override */
 723                 if (!huge)
 724                         cur_pgsz = sysconf(_SC_PAGESIZE);
 725
 726                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 727                 if (ret < 0) {
 728                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 729                         return -1;
 730                 }
 731
 732                 /* allocate our memory */
 733                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 734
 735                 /* if we couldn't allocate memory with a specified page size,
 736                  * that doesn't mean we can't do it with other page sizes, so
 737                  * try another one.
 738                  */
 739                 if (addr == NULL)
 740                         continue;
 741
 742                 /* store IOVA addresses for every page in this memory area */
 743                 n_pages = mem_sz / cur_pgsz;
 744
 745                 iovas = malloc(sizeof(*iovas) * n_pages);
 746
 747                 if (iovas == NULL) {
 748                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 749                         goto fail;
 750                 }
 751                 /* lock memory if it's not huge pages */
 752                 if (!huge)
 753                         mlock(addr, mem_sz);
 754
 755                 /* populate IOVA addresses */
 756                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 757                         rte_iova_t iova;
 758                         size_t offset;
 759                         void *cur;
 760
 761                         offset = cur_pgsz * cur_page;
 762                         cur = RTE_PTR_ADD(addr, offset);
 763
 764                         /* touch the page before getting its IOVA */
 765                         *(volatile char *)cur = 0;
 766
 767                         iova = rte_mem_virt2iova(cur);
 768
 769                         iovas[cur_page] = iova;
 770                 }
 771
 772                 break;
 773         }
 774         /* if we couldn't allocate anything */
 775         if (iovas == NULL)
 776                 return -1;
 777
 778         param->addr = addr;
 779         param->len = mem_sz;
 780         param->pgsz = cur_pgsz;
 781         param->iova_table = iovas;
 782         param->iova_table_len = n_pages;
 783
 784         return 0;
 785 fail:
 786         if (iovas)
 787                 free(iovas);
 788         if (addr)
 789                 munmap(addr, mem_sz);
 790
 791         return -1;
 792 }
 793
 794 static int
 795 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 796 {
 797         struct extmem_param param;
 798         int socket_id, ret;
 799
 800         memset(&param, 0, sizeof(param));
 801
 802         /* check if our heap exists */
 803         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 804         if (socket_id < 0) {
 805                 /* create our heap */
 806                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 807                 if (ret < 0) {
 808                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 809                         return -1;
 810                 }
 811         }
 812
 813         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 814         if (ret < 0) {
 815                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 816                 return -1;
 817         }
 818
 819         /* we now have a valid memory area, so add it to heap */
 820         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 821                         param.addr, param.len, param.iova_table,
 822                         param.iova_table_len, param.pgsz);
 823
 824         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 825
 826         /* not needed any more */
 827         free(param.iova_table);
 828
 829         if (ret < 0) {
 830                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 831                 munmap(param.addr, param.len);
 832                 return -1;
 833         }
 834
 835         /* success */
 836
 837         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 838                         param.len >> 20);
 839
 840         return 0;
 841 }
 842 static void
 843 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 844              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 845 {
 846         uint16_t pid = 0;
 847         int ret;
 848
 849         RTE_ETH_FOREACH_DEV(pid) {
 850                 struct rte_eth_dev *dev =
 851                         &rte_eth_devices[pid];
 852
 853                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 854                                         memhdr->len);
 855                 if (ret) {
 856                         TESTPMD_LOG(DEBUG,
 857                                     "unable to DMA unmap addr 0x%p "
 858                                     "for device %s\n",
 859                                     memhdr->addr, dev->data->name);
 860                 }
 861         }
 862         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 863         if (ret) {
 864                 TESTPMD_LOG(DEBUG,
 865                             "unable to un-register addr 0x%p\n", memhdr->addr);
 866         }
 867 }
 868
 869 static void
 870 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 871            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 872 {
 873         uint16_t pid = 0;
 874         size_t page_size = sysconf(_SC_PAGESIZE);
 875         int ret;
 876
 877         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 878                                   page_size);
 879         if (ret) {
 880                 TESTPMD_LOG(DEBUG,
 881                             "unable to register addr 0x%p\n", memhdr->addr);
 882                 return;
 883         }
 884         RTE_ETH_FOREACH_DEV(pid) {
 885                 struct rte_eth_dev *dev =
 886                         &rte_eth_devices[pid];
 887
 888                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 889                                       memhdr->len);
 890                 if (ret) {
 891                         TESTPMD_LOG(DEBUG,
 892                                     "unable to DMA map addr 0x%p "
 893                                     "for device %s\n",
 894                                     memhdr->addr, dev->data->name);
 895                 }
 896         }
 897 }
 898
 899 static unsigned int
 900 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 901             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 902 {
 903         struct rte_pktmbuf_extmem *xmem;
 904         unsigned int ext_num, zone_num, elt_num;
 905         uint16_t elt_size;
 906
 907         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 908         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 909         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 910
 911         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 912         if (xmem == NULL) {
 913                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 914                                  "external buffer descriptors\n");
 915                 *ext_mem = NULL;
 916                 return 0;
 917         }
 918         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 919                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 920                 const struct rte_memzone *mz;
 921                 char mz_name[RTE_MEMZONE_NAMESIZE];
 922                 int ret;
 923
 924                 ret = snprintf(mz_name, sizeof(mz_name),
 925                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 926                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 927                         errno = ENAMETOOLONG;
 928                         ext_num = 0;
 929                         break;
 930                 }
 931                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 932                                                  socket_id,
 933                                                  RTE_MEMZONE_IOVA_CONTIG |
 934                                                  RTE_MEMZONE_1GB |
 935                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 936                                                  EXTBUF_ZONE_SIZE);
 937                 if (mz == NULL) {
 938                         /*
 939                          * The caller exits on external buffer creation
 940                          * error, so there is no need to free memzones.
 941                          */
 942                         errno = ENOMEM;
 943                         ext_num = 0;
 944                         break;
 945                 }
 946                 xseg->buf_ptr = mz->addr;
 947                 xseg->buf_iova = mz->iova;
 948                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 949                 xseg->elt_size = elt_size;
 950         }
 951         if (ext_num == 0 && xmem != NULL) {
 952                 free(xmem);
 953                 xmem = NULL;
 954         }
 955         *ext_mem = xmem;
 956         return ext_num;
 957 }
 958
 959 /*
 960  * Configuration initialisation done once at init time.
 961  */
 962 static struct rte_mempool *
 963 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 964                  unsigned int socket_id, uint16_t size_idx)
 965 {
 966         char pool_name[RTE_MEMPOOL_NAMESIZE];
 967         struct rte_mempool *rte_mp = NULL;
 968         uint32_t mb_size;
 969
 970         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 971         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 972
 973         TESTPMD_LOG(INFO,
 974                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 975                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 976
 977         switch (mp_alloc_type) {
 978         case MP_ALLOC_NATIVE:
 979                 {
 980                         /* wrapper to rte_mempool_create() */
 981                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 982                                         rte_mbuf_best_mempool_ops());
 983                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 984                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 985                         break;
 986                 }
 987         case MP_ALLOC_ANON:
 988                 {
 989                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 990                                 mb_size, (unsigned int) mb_mempool_cache,
 991                                 sizeof(struct rte_pktmbuf_pool_private),
 992                                 socket_id, mempool_flags);
 993                         if (rte_mp == NULL)
 994                                 goto err;
 995
 996                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 997                                 rte_mempool_free(rte_mp);
 998                                 rte_mp = NULL;
 999                                 goto err;
1000                         }
1001                         rte_pktmbuf_pool_init(rte_mp, NULL);
1002                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1003                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1004                         break;
1005                 }
1006         case MP_ALLOC_XMEM:
1007         case MP_ALLOC_XMEM_HUGE:
1008                 {
1009                         int heap_socket;
1010                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1011
1012                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1013                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1014
1015                         heap_socket =
1016                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1017                         if (heap_socket < 0)
1018                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1019
1020                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1021                                         rte_mbuf_best_mempool_ops());
1022                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1023                                         mb_mempool_cache, 0, mbuf_seg_size,
1024                                         heap_socket);
1025                         break;
1026                 }
1027         case MP_ALLOC_XBUF:
1028                 {
1029                         struct rte_pktmbuf_extmem *ext_mem;
1030                         unsigned int ext_num;
1031
1032                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1033                                                socket_id, pool_name, &ext_mem);
1034                         if (ext_num == 0)
1035                                 rte_exit(EXIT_FAILURE,
1036                                          "Can't create pinned data buffers\n");
1037
1038                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1039                                         rte_mbuf_best_mempool_ops());
1040                         rte_mp = rte_pktmbuf_pool_create_extbuf
1041                                         (pool_name, nb_mbuf, mb_mempool_cache,
1042                                          0, mbuf_seg_size, socket_id,
1043                                          ext_mem, ext_num);
1044                         free(ext_mem);
1045                         break;
1046                 }
1047         default:
1048                 {
1049                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1050                 }
1051         }
1052
1053 err:
1054         if (rte_mp == NULL) {
1055                 rte_exit(EXIT_FAILURE,
1056                         "Creation of mbuf pool for socket %u failed: %s\n",
1057                         socket_id, rte_strerror(rte_errno));
1058         } else if (verbose_level > 0) {
1059                 rte_mempool_dump(stdout, rte_mp);
1060         }
1061         return rte_mp;
1062 }
1063
1064 /*
1065  * Check given socket id is valid or not with NUMA mode,
1066  * if valid, return 0, else return -1
1067  */
1068 static int
1069 check_socket_id(const unsigned int socket_id)
1070 {
1071         static int warning_once = 0;
1072
1073         if (new_socket_id(socket_id)) {
1074                 if (!warning_once && numa_support)
1075                         printf("Warning: NUMA should be configured manually by"
1076                                " using --port-numa-config and"
1077                                " --ring-numa-config parameters along with"
1078                                " --numa.\n");
1079                 warning_once = 1;
1080                 return -1;
1081         }
1082         return 0;
1083 }
1084
1085 /*
1086  * Get the allowed maximum number of RX queues.
1087  * *pid return the port id which has minimal value of
1088  * max_rx_queues in all ports.
1089  */
1090 queueid_t
1091 get_allowed_max_nb_rxq(portid_t *pid)
1092 {
1093         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1094         bool max_rxq_valid = false;
1095         portid_t pi;
1096         struct rte_eth_dev_info dev_info;
1097
1098         RTE_ETH_FOREACH_DEV(pi) {
1099                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1100                         continue;
1101
1102                 max_rxq_valid = true;
1103                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1104                         allowed_max_rxq = dev_info.max_rx_queues;
1105                         *pid = pi;
1106                 }
1107         }
1108         return max_rxq_valid ? allowed_max_rxq : 0;
1109 }
1110
1111 /*
1112  * Check input rxq is valid or not.
1113  * If input rxq is not greater than any of maximum number
1114  * of RX queues of all ports, it is valid.
1115  * if valid, return 0, else return -1
1116  */
1117 int
1118 check_nb_rxq(queueid_t rxq)
1119 {
1120         queueid_t allowed_max_rxq;
1121         portid_t pid = 0;
1122
1123         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1124         if (rxq > allowed_max_rxq) {
1125                 printf("Fail: input rxq (%u) can't be greater "
1126                        "than max_rx_queues (%u) of port %u\n",
1127                        rxq,
1128                        allowed_max_rxq,
1129                        pid);
1130                 return -1;
1131         }
1132         return 0;
1133 }
1134
1135 /*
1136  * Get the allowed maximum number of TX queues.
1137  * *pid return the port id which has minimal value of
1138  * max_tx_queues in all ports.
1139  */
1140 queueid_t
1141 get_allowed_max_nb_txq(portid_t *pid)
1142 {
1143         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1144         bool max_txq_valid = false;
1145         portid_t pi;
1146         struct rte_eth_dev_info dev_info;
1147
1148         RTE_ETH_FOREACH_DEV(pi) {
1149                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1150                         continue;
1151
1152                 max_txq_valid = true;
1153                 if (dev_info.max_tx_queues < allowed_max_txq) {
1154                         allowed_max_txq = dev_info.max_tx_queues;
1155                         *pid = pi;
1156                 }
1157         }
1158         return max_txq_valid ? allowed_max_txq : 0;
1159 }
1160
1161 /*
1162  * Check input txq is valid or not.
1163  * If input txq is not greater than any of maximum number
1164  * of TX queues of all ports, it is valid.
1165  * if valid, return 0, else return -1
1166  */
1167 int
1168 check_nb_txq(queueid_t txq)
1169 {
1170         queueid_t allowed_max_txq;
1171         portid_t pid = 0;
1172
1173         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1174         if (txq > allowed_max_txq) {
1175                 printf("Fail: input txq (%u) can't be greater "
1176                        "than max_tx_queues (%u) of port %u\n",
1177                        txq,
1178                        allowed_max_txq,
1179                        pid);
1180                 return -1;
1181         }
1182         return 0;
1183 }
1184
1185 /*
1186  * Get the allowed maximum number of RXDs of every rx queue.
1187  * *pid return the port id which has minimal value of
1188  * max_rxd in all queues of all ports.
1189  */
1190 static uint16_t
1191 get_allowed_max_nb_rxd(portid_t *pid)
1192 {
1193         uint16_t allowed_max_rxd = UINT16_MAX;
1194         portid_t pi;
1195         struct rte_eth_dev_info dev_info;
1196
1197         RTE_ETH_FOREACH_DEV(pi) {
1198                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1199                         continue;
1200
1201                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1202                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1203                         *pid = pi;
1204                 }
1205         }
1206         return allowed_max_rxd;
1207 }
1208
1209 /*
1210  * Get the allowed minimal number of RXDs of every rx queue.
1211  * *pid return the port id which has minimal value of
1212  * min_rxd in all queues of all ports.
1213  */
1214 static uint16_t
1215 get_allowed_min_nb_rxd(portid_t *pid)
1216 {
1217         uint16_t allowed_min_rxd = 0;
1218         portid_t pi;
1219         struct rte_eth_dev_info dev_info;
1220
1221         RTE_ETH_FOREACH_DEV(pi) {
1222                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1223                         continue;
1224
1225                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1226                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1227                         *pid = pi;
1228                 }
1229         }
1230
1231         return allowed_min_rxd;
1232 }
1233
1234 /*
1235  * Check input rxd is valid or not.
1236  * If input rxd is not greater than any of maximum number
1237  * of RXDs of every Rx queues and is not less than any of
1238  * minimal number of RXDs of every Rx queues, it is valid.
1239  * if valid, return 0, else return -1
1240  */
1241 int
1242 check_nb_rxd(queueid_t rxd)
1243 {
1244         uint16_t allowed_max_rxd;
1245         uint16_t allowed_min_rxd;
1246         portid_t pid = 0;
1247
1248         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1249         if (rxd > allowed_max_rxd) {
1250                 printf("Fail: input rxd (%u) can't be greater "
1251                        "than max_rxds (%u) of port %u\n",
1252                        rxd,
1253                        allowed_max_rxd,
1254                        pid);
1255                 return -1;
1256         }
1257
1258         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1259         if (rxd < allowed_min_rxd) {
1260                 printf("Fail: input rxd (%u) can't be less "
1261                        "than min_rxds (%u) of port %u\n",
1262                        rxd,
1263                        allowed_min_rxd,
1264                        pid);
1265                 return -1;
1266         }
1267
1268         return 0;
1269 }
1270
1271 /*
1272  * Get the allowed maximum number of TXDs of every rx queues.
1273  * *pid return the port id which has minimal value of
1274  * max_txd in every tx queue.
1275  */
1276 static uint16_t
1277 get_allowed_max_nb_txd(portid_t *pid)
1278 {
1279         uint16_t allowed_max_txd = UINT16_MAX;
1280         portid_t pi;
1281         struct rte_eth_dev_info dev_info;
1282
1283         RTE_ETH_FOREACH_DEV(pi) {
1284                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1285                         continue;
1286
1287                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1288                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1289                         *pid = pi;
1290                 }
1291         }
1292         return allowed_max_txd;
1293 }
1294
1295 /*
1296  * Get the allowed maximum number of TXDs of every tx queues.
1297  * *pid return the port id which has minimal value of
1298  * min_txd in every tx queue.
1299  */
1300 static uint16_t
1301 get_allowed_min_nb_txd(portid_t *pid)
1302 {
1303         uint16_t allowed_min_txd = 0;
1304         portid_t pi;
1305         struct rte_eth_dev_info dev_info;
1306
1307         RTE_ETH_FOREACH_DEV(pi) {
1308                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1309                         continue;
1310
1311                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1312                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1313                         *pid = pi;
1314                 }
1315         }
1316
1317         return allowed_min_txd;
1318 }
1319
1320 /*
1321  * Check input txd is valid or not.
1322  * If input txd is not greater than any of maximum number
1323  * of TXDs of every Rx queues, it is valid.
1324  * if valid, return 0, else return -1
1325  */
1326 int
1327 check_nb_txd(queueid_t txd)
1328 {
1329         uint16_t allowed_max_txd;
1330         uint16_t allowed_min_txd;
1331         portid_t pid = 0;
1332
1333         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1334         if (txd > allowed_max_txd) {
1335                 printf("Fail: input txd (%u) can't be greater "
1336                        "than max_txds (%u) of port %u\n",
1337                        txd,
1338                        allowed_max_txd,
1339                        pid);
1340                 return -1;
1341         }
1342
1343         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1344         if (txd < allowed_min_txd) {
1345                 printf("Fail: input txd (%u) can't be less "
1346                        "than min_txds (%u) of port %u\n",
1347                        txd,
1348                        allowed_min_txd,
1349                        pid);
1350                 return -1;
1351         }
1352         return 0;
1353 }
1354
1355
1356 /*
1357  * Get the allowed maximum number of hairpin queues.
1358  * *pid return the port id which has minimal value of
1359  * max_hairpin_queues in all ports.
1360  */
1361 queueid_t
1362 get_allowed_max_nb_hairpinq(portid_t *pid)
1363 {
1364         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1365         portid_t pi;
1366         struct rte_eth_hairpin_cap cap;
1367
1368         RTE_ETH_FOREACH_DEV(pi) {
1369                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1370                         *pid = pi;
1371                         return 0;
1372                 }
1373                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1374                         allowed_max_hairpinq = cap.max_nb_queues;
1375                         *pid = pi;
1376                 }
1377         }
1378         return allowed_max_hairpinq;
1379 }
1380
1381 /*
1382  * Check input hairpin is valid or not.
1383  * If input hairpin is not greater than any of maximum number
1384  * of hairpin queues of all ports, it is valid.
1385  * if valid, return 0, else return -1
1386  */
1387 int
1388 check_nb_hairpinq(queueid_t hairpinq)
1389 {
1390         queueid_t allowed_max_hairpinq;
1391         portid_t pid = 0;
1392
1393         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1394         if (hairpinq > allowed_max_hairpinq) {
1395                 printf("Fail: input hairpin (%u) can't be greater "
1396                        "than max_hairpin_queues (%u) of port %u\n",
1397                        hairpinq, allowed_max_hairpinq, pid);
1398                 return -1;
1399         }
1400         return 0;
1401 }
1402
1403 static void
1404 init_config(void)
1405 {
1406         portid_t pid;
1407         struct rte_port *port;
1408         struct rte_mempool *mbp;
1409         unsigned int nb_mbuf_per_pool;
1410         lcoreid_t  lc_id;
1411         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1412         struct rte_gro_param gro_param;
1413         uint32_t gso_types;
1414         uint16_t data_size;
1415         bool warning = 0;
1416         int k;
1417         int ret;
1418
1419         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1420
1421         /* Configuration of logical cores. */
1422         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1423                                 sizeof(struct fwd_lcore *) * nb_lcores,
1424                                 RTE_CACHE_LINE_SIZE);
1425         if (fwd_lcores == NULL) {
1426                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1427                                                         "failed\n", nb_lcores);
1428         }
1429         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1430                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1431                                                sizeof(struct fwd_lcore),
1432                                                RTE_CACHE_LINE_SIZE);
1433                 if (fwd_lcores[lc_id] == NULL) {
1434                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1435                                                                 "failed\n");
1436                 }
1437                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1438         }
1439
1440         RTE_ETH_FOREACH_DEV(pid) {
1441                 port = &ports[pid];
1442                 /* Apply default TxRx configuration for all ports */
1443                 port->dev_conf.txmode = tx_mode;
1444                 port->dev_conf.rxmode = rx_mode;
1445
1446                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1447                 if (ret != 0)
1448                         rte_exit(EXIT_FAILURE,
1449                                  "rte_eth_dev_info_get() failed\n");
1450
1451                 if (!(port->dev_info.tx_offload_capa &
1452                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1453                         port->dev_conf.txmode.offloads &=
1454                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1455                 if (numa_support) {
1456                         if (port_numa[pid] != NUMA_NO_CONFIG)
1457                                 port_per_socket[port_numa[pid]]++;
1458                         else {
1459                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1460
1461                                 /*
1462                                  * if socket_id is invalid,
1463                                  * set to the first available socket.
1464                                  */
1465                                 if (check_socket_id(socket_id) < 0)
1466                                         socket_id = socket_ids[0];
1467                                 port_per_socket[socket_id]++;
1468                         }
1469                 }
1470
1471                 /* Apply Rx offloads configuration */
1472                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1473                         port->rx_conf[k].offloads =
1474                                 port->dev_conf.rxmode.offloads;
1475                 /* Apply Tx offloads configuration */
1476                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1477                         port->tx_conf[k].offloads =
1478                                 port->dev_conf.txmode.offloads;
1479
1480                 /* set flag to initialize port/queue */
1481                 port->need_reconfig = 1;
1482                 port->need_reconfig_queues = 1;
1483                 port->tx_metadata = 0;
1484
1485                 /* Check for maximum number of segments per MTU. Accordingly
1486                  * update the mbuf data size.
1487                  */
1488                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1489                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1490                         data_size = rx_mode.max_rx_pkt_len /
1491                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1492
1493                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1494                                                         mbuf_data_size[0]) {
1495                                 mbuf_data_size[0] = data_size +
1496                                                  RTE_PKTMBUF_HEADROOM;
1497                                 warning = 1;
1498                         }
1499                 }
1500         }
1501
1502         if (warning)
1503                 TESTPMD_LOG(WARNING,
1504                             "Configured mbuf size of the first segment %hu\n",
1505                             mbuf_data_size[0]);
1506         /*
1507          * Create pools of mbuf.
1508          * If NUMA support is disabled, create a single pool of mbuf in
1509          * socket 0 memory by default.
1510          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1511          *
1512          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1513          * nb_txd can be configured at run time.
1514          */
1515         if (param_total_num_mbufs)
1516                 nb_mbuf_per_pool = param_total_num_mbufs;
1517         else {
1518                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1519                         (nb_lcores * mb_mempool_cache) +
1520                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1521                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1522         }
1523
1524         if (numa_support) {
1525                 uint8_t i, j;
1526
1527                 for (i = 0; i < num_sockets; i++)
1528                         for (j = 0; j < mbuf_data_size_n; j++)
1529                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1530                                         mbuf_pool_create(mbuf_data_size[j],
1531                                                           nb_mbuf_per_pool,
1532                                                           socket_ids[i], j);
1533         } else {
1534                 uint8_t i;
1535
1536                 for (i = 0; i < mbuf_data_size_n; i++)
1537                         mempools[i] = mbuf_pool_create
1538                                         (mbuf_data_size[i],
1539                                          nb_mbuf_per_pool,
1540                                          socket_num == UMA_NO_CONFIG ?
1541                                          0 : socket_num, i);
1542         }
1543
1544         init_port_config();
1545
1546         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1547                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1548         /*
1549          * Records which Mbuf pool to use by each logical core, if needed.
1550          */
1551         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1552                 mbp = mbuf_pool_find(
1553                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1554
1555                 if (mbp == NULL)
1556                         mbp = mbuf_pool_find(0, 0);
1557                 fwd_lcores[lc_id]->mbp = mbp;
1558                 /* initialize GSO context */
1559                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1560                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1561                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1562                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1563                         RTE_ETHER_CRC_LEN;
1564                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1565         }
1566
1567         /* Configuration of packet forwarding streams. */
1568         if (init_fwd_streams() < 0)
1569                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1570
1571         fwd_config_setup();
1572
1573         /* create a gro context for each lcore */
1574         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1575         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1576         gro_param.max_item_per_flow = MAX_PKT_BURST;
1577         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1578                 gro_param.socket_id = rte_lcore_to_socket_id(
1579                                 fwd_lcores_cpuids[lc_id]);
1580                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1581                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1582                         rte_exit(EXIT_FAILURE,
1583                                         "rte_gro_ctx_create() failed\n");
1584                 }
1585         }
1586 }
1587
1588
1589 void
1590 reconfig(portid_t new_port_id, unsigned socket_id)
1591 {
1592         struct rte_port *port;
1593         int ret;
1594
1595         /* Reconfiguration of Ethernet ports. */
1596         port = &ports[new_port_id];
1597
1598         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1599         if (ret != 0)
1600                 return;
1601
1602         /* set flag to initialize port/queue */
1603         port->need_reconfig = 1;
1604         port->need_reconfig_queues = 1;
1605         port->socket_id = socket_id;
1606
1607         init_port_config();
1608 }
1609
1610
1611 int
1612 init_fwd_streams(void)
1613 {
1614         portid_t pid;
1615         struct rte_port *port;
1616         streamid_t sm_id, nb_fwd_streams_new;
1617         queueid_t q;
1618
1619         /* set socket id according to numa or not */
1620         RTE_ETH_FOREACH_DEV(pid) {
1621                 port = &ports[pid];
1622                 if (nb_rxq > port->dev_info.max_rx_queues) {
1623                         printf("Fail: nb_rxq(%d) is greater than "
1624                                 "max_rx_queues(%d)\n", nb_rxq,
1625                                 port->dev_info.max_rx_queues);
1626                         return -1;
1627                 }
1628                 if (nb_txq > port->dev_info.max_tx_queues) {
1629                         printf("Fail: nb_txq(%d) is greater than "
1630                                 "max_tx_queues(%d)\n", nb_txq,
1631                                 port->dev_info.max_tx_queues);
1632                         return -1;
1633                 }
1634                 if (numa_support) {
1635                         if (port_numa[pid] != NUMA_NO_CONFIG)
1636                                 port->socket_id = port_numa[pid];
1637                         else {
1638                                 port->socket_id = rte_eth_dev_socket_id(pid);
1639
1640                                 /*
1641                                  * if socket_id is invalid,
1642                                  * set to the first available socket.
1643                                  */
1644                                 if (check_socket_id(port->socket_id) < 0)
1645                                         port->socket_id = socket_ids[0];
1646                         }
1647                 }
1648                 else {
1649                         if (socket_num == UMA_NO_CONFIG)
1650                                 port->socket_id = 0;
1651                         else
1652                                 port->socket_id = socket_num;
1653                 }
1654         }
1655
1656         q = RTE_MAX(nb_rxq, nb_txq);
1657         if (q == 0) {
1658                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1659                 return -1;
1660         }
1661         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1662         if (nb_fwd_streams_new == nb_fwd_streams)
1663                 return 0;
1664         /* clear the old */
1665         if (fwd_streams != NULL) {
1666                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1667                         if (fwd_streams[sm_id] == NULL)
1668                                 continue;
1669                         rte_free(fwd_streams[sm_id]);
1670                         fwd_streams[sm_id] = NULL;
1671                 }
1672                 rte_free(fwd_streams);
1673                 fwd_streams = NULL;
1674         }
1675
1676         /* init new */
1677         nb_fwd_streams = nb_fwd_streams_new;
1678         if (nb_fwd_streams) {
1679                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1680                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1681                         RTE_CACHE_LINE_SIZE);
1682                 if (fwd_streams == NULL)
1683                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1684                                  " (struct fwd_stream *)) failed\n",
1685                                  nb_fwd_streams);
1686
1687                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1688                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1689                                 " struct fwd_stream", sizeof(struct fwd_stream),
1690                                 RTE_CACHE_LINE_SIZE);
1691                         if (fwd_streams[sm_id] == NULL)
1692                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1693                                          "(struct fwd_stream) failed\n");
1694                 }
1695         }
1696
1697         return 0;
1698 }
1699
1700 static void
1701 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1702 {
1703         uint64_t total_burst, sburst;
1704         uint64_t nb_burst;
1705         uint64_t burst_stats[4];
1706         uint16_t pktnb_stats[4];
1707         uint16_t nb_pkt;
1708         int burst_percent[4], sburstp;
1709         int i;
1710
1711         /*
1712          * First compute the total number of packet bursts and the
1713          * two highest numbers of bursts of the same number of packets.
1714          */
1715         memset(&burst_stats, 0x0, sizeof(burst_stats));
1716         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1717
1718         /* Show stats for 0 burst size always */
1719         total_burst = pbs->pkt_burst_spread[0];
1720         burst_stats[0] = pbs->pkt_burst_spread[0];
1721         pktnb_stats[0] = 0;
1722
1723         /* Find the next 2 burst sizes with highest occurrences. */
1724         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1725                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1726
1727                 if (nb_burst == 0)
1728                         continue;
1729
1730                 total_burst += nb_burst;
1731
1732                 if (nb_burst > burst_stats[1]) {
1733                         burst_stats[2] = burst_stats[1];
1734                         pktnb_stats[2] = pktnb_stats[1];
1735                         burst_stats[1] = nb_burst;
1736                         pktnb_stats[1] = nb_pkt;
1737                 } else if (nb_burst > burst_stats[2]) {
1738                         burst_stats[2] = nb_burst;
1739                         pktnb_stats[2] = nb_pkt;
1740                 }
1741         }
1742         if (total_burst == 0)
1743                 return;
1744
1745         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1746         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1747                 if (i == 3) {
1748                         printf("%d%% of other]\n", 100 - sburstp);
1749                         return;
1750                 }
1751
1752                 sburst += burst_stats[i];
1753                 if (sburst == total_burst) {
1754                         printf("%d%% of %d pkts]\n",
1755                                 100 - sburstp, (int) pktnb_stats[i]);
1756                         return;
1757                 }
1758
1759                 burst_percent[i] =
1760                         (double)burst_stats[i] / total_burst * 100;
1761                 printf("%d%% of %d pkts + ",
1762                         burst_percent[i], (int) pktnb_stats[i]);
1763                 sburstp += burst_percent[i];
1764         }
1765 }
1766
1767 static void
1768 fwd_stream_stats_display(streamid_t stream_id)
1769 {
1770         struct fwd_stream *fs;
1771         static const char *fwd_top_stats_border = "-------";
1772
1773         fs = fwd_streams[stream_id];
1774         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1775             (fs->fwd_dropped == 0))
1776                 return;
1777         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1778                "TX Port=%2d/Queue=%2d %s\n",
1779                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1780                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1781         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1782                " TX-dropped: %-14"PRIu64,
1783                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1784
1785         /* if checksum mode */
1786         if (cur_fwd_eng == &csum_fwd_engine) {
1787                 printf("  RX- bad IP checksum: %-14"PRIu64
1788                        "  Rx- bad L4 checksum: %-14"PRIu64
1789                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1790                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1791                         fs->rx_bad_outer_l4_csum);
1792         } else {
1793                 printf("\n");
1794         }
1795
1796         if (record_burst_stats) {
1797                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1798                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1799         }
1800 }
1801
1802 void
1803 fwd_stats_display(void)
1804 {
1805         static const char *fwd_stats_border = "----------------------";
1806         static const char *acc_stats_border = "+++++++++++++++";
1807         struct {
1808                 struct fwd_stream *rx_stream;
1809                 struct fwd_stream *tx_stream;
1810                 uint64_t tx_dropped;
1811                 uint64_t rx_bad_ip_csum;
1812                 uint64_t rx_bad_l4_csum;
1813                 uint64_t rx_bad_outer_l4_csum;
1814         } ports_stats[RTE_MAX_ETHPORTS];
1815         uint64_t total_rx_dropped = 0;
1816         uint64_t total_tx_dropped = 0;
1817         uint64_t total_rx_nombuf = 0;
1818         struct rte_eth_stats stats;
1819         uint64_t fwd_cycles = 0;
1820         uint64_t total_recv = 0;
1821         uint64_t total_xmit = 0;
1822         struct rte_port *port;
1823         streamid_t sm_id;
1824         portid_t pt_id;
1825         int i;
1826
1827         memset(ports_stats, 0, sizeof(ports_stats));
1828
1829         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1830                 struct fwd_stream *fs = fwd_streams[sm_id];
1831
1832                 if (cur_fwd_config.nb_fwd_streams >
1833                     cur_fwd_config.nb_fwd_ports) {
1834                         fwd_stream_stats_display(sm_id);
1835                 } else {
1836                         ports_stats[fs->tx_port].tx_stream = fs;
1837                         ports_stats[fs->rx_port].rx_stream = fs;
1838                 }
1839
1840                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1841
1842                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1843                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1844                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1845                                 fs->rx_bad_outer_l4_csum;
1846
1847                 if (record_core_cycles)
1848                         fwd_cycles += fs->core_cycles;
1849         }
1850         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1851                 uint8_t j;
1852
1853                 pt_id = fwd_ports_ids[i];
1854                 port = &ports[pt_id];
1855
1856                 rte_eth_stats_get(pt_id, &stats);
1857                 stats.ipackets -= port->stats.ipackets;
1858                 stats.opackets -= port->stats.opackets;
1859                 stats.ibytes -= port->stats.ibytes;
1860                 stats.obytes -= port->stats.obytes;
1861                 stats.imissed -= port->stats.imissed;
1862                 stats.oerrors -= port->stats.oerrors;
1863                 stats.rx_nombuf -= port->stats.rx_nombuf;
1864
1865                 total_recv += stats.ipackets;
1866                 total_xmit += stats.opackets;
1867                 total_rx_dropped += stats.imissed;
1868                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1869                 total_tx_dropped += stats.oerrors;
1870                 total_rx_nombuf  += stats.rx_nombuf;
1871
1872                 printf("\n  %s Forward statistics for port %-2d %s\n",
1873                        fwd_stats_border, pt_id, fwd_stats_border);
1874
1875                 if (!port->rx_queue_stats_mapping_enabled &&
1876                     !port->tx_queue_stats_mapping_enabled) {
1877                         printf("  RX-packets: %-14"PRIu64
1878                                " RX-dropped: %-14"PRIu64
1879                                "RX-total: %-"PRIu64"\n",
1880                                stats.ipackets, stats.imissed,
1881                                stats.ipackets + stats.imissed);
1882
1883                         if (cur_fwd_eng == &csum_fwd_engine)
1884                                 printf("  Bad-ipcsum: %-14"PRIu64
1885                                        " Bad-l4csum: %-14"PRIu64
1886                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1887                                        ports_stats[pt_id].rx_bad_ip_csum,
1888                                        ports_stats[pt_id].rx_bad_l4_csum,
1889                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1890                         if (stats.ierrors + stats.rx_nombuf > 0) {
1891                                 printf("  RX-error: %-"PRIu64"\n",
1892                                        stats.ierrors);
1893                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1894                                        stats.rx_nombuf);
1895                         }
1896
1897                         printf("  TX-packets: %-14"PRIu64
1898                                " TX-dropped: %-14"PRIu64
1899                                "TX-total: %-"PRIu64"\n",
1900                                stats.opackets, ports_stats[pt_id].tx_dropped,
1901                                stats.opackets + ports_stats[pt_id].tx_dropped);
1902                 } else {
1903                         printf("  RX-packets:             %14"PRIu64
1904                                "    RX-dropped:%14"PRIu64
1905                                "    RX-total:%14"PRIu64"\n",
1906                                stats.ipackets, stats.imissed,
1907                                stats.ipackets + stats.imissed);
1908
1909                         if (cur_fwd_eng == &csum_fwd_engine)
1910                                 printf("  Bad-ipcsum:%14"PRIu64
1911                                        "    Bad-l4csum:%14"PRIu64
1912                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1913                                        ports_stats[pt_id].rx_bad_ip_csum,
1914                                        ports_stats[pt_id].rx_bad_l4_csum,
1915                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1916                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1917                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1918                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1919                                        stats.rx_nombuf);
1920                         }
1921
1922                         printf("  TX-packets:             %14"PRIu64
1923                                "    TX-dropped:%14"PRIu64
1924                                "    TX-total:%14"PRIu64"\n",
1925                                stats.opackets, ports_stats[pt_id].tx_dropped,
1926                                stats.opackets + ports_stats[pt_id].tx_dropped);
1927                 }
1928
1929                 if (record_burst_stats) {
1930                         if (ports_stats[pt_id].rx_stream)
1931                                 pkt_burst_stats_display("RX",
1932                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1933                         if (ports_stats[pt_id].tx_stream)
1934                                 pkt_burst_stats_display("TX",
1935                                         &ports_stats[pt_id].tx_stream->tx_burst_stats);
1936                 }
1937
1938                 if (port->rx_queue_stats_mapping_enabled) {
1939                         printf("\n");
1940                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1941                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1942                                        "     RX-errors:%14"PRIu64
1943                                        "    RX-bytes:%14"PRIu64"\n",
1944                                        j, stats.q_ipackets[j],
1945                                        stats.q_errors[j], stats.q_ibytes[j]);
1946                         }
1947                         printf("\n");
1948                 }
1949                 if (port->tx_queue_stats_mapping_enabled) {
1950                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1951                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1952                                        "                                 TX-bytes:%14"
1953                                        PRIu64"\n",
1954                                        j, stats.q_opackets[j],
1955                                        stats.q_obytes[j]);
1956                         }
1957                 }
1958
1959                 printf("  %s--------------------------------%s\n",
1960                        fwd_stats_border, fwd_stats_border);
1961         }
1962
1963         printf("\n  %s Accumulated forward statistics for all ports"
1964                "%s\n",
1965                acc_stats_border, acc_stats_border);
1966         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1967                "%-"PRIu64"\n"
1968                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1969                "%-"PRIu64"\n",
1970                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1971                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1972         if (total_rx_nombuf > 0)
1973                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1974         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1975                "%s\n",
1976                acc_stats_border, acc_stats_border);
1977         if (record_core_cycles) {
1978 #define CYC_PER_MHZ 1E6
1979                 if (total_recv > 0 || total_xmit > 0) {
1980                         uint64_t total_pkts = 0;
1981                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1982                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1983                                 total_pkts = total_xmit;
1984                         else
1985                                 total_pkts = total_recv;
1986
1987                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1988                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1989                                " MHz Clock\n",
1990                                (double) fwd_cycles / total_pkts,
1991                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1992                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1993                 }
1994         }
1995 }
1996
1997 void
1998 fwd_stats_reset(void)
1999 {
2000         streamid_t sm_id;
2001         portid_t pt_id;
2002         int i;
2003
2004         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2005                 pt_id = fwd_ports_ids[i];
2006                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2007         }
2008         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2009                 struct fwd_stream *fs = fwd_streams[sm_id];
2010
2011                 fs->rx_packets = 0;
2012                 fs->tx_packets = 0;
2013                 fs->fwd_dropped = 0;
2014                 fs->rx_bad_ip_csum = 0;
2015                 fs->rx_bad_l4_csum = 0;
2016                 fs->rx_bad_outer_l4_csum = 0;
2017
2018                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2019                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2020                 fs->core_cycles = 0;
2021         }
2022 }
2023
2024 static void
2025 flush_fwd_rx_queues(void)
2026 {
2027         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2028         portid_t  rxp;
2029         portid_t port_id;
2030         queueid_t rxq;
2031         uint16_t  nb_rx;
2032         uint16_t  i;
2033         uint8_t   j;
2034         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2035         uint64_t timer_period;
2036
2037         /* convert to number of cycles */
2038         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2039
2040         for (j = 0; j < 2; j++) {
2041                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2042                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2043                                 port_id = fwd_ports_ids[rxp];
2044                                 /**
2045                                 * testpmd can stuck in the below do while loop
2046                                 * if rte_eth_rx_burst() always returns nonzero
2047                                 * packets. So timer is added to exit this loop
2048                                 * after 1sec timer expiry.
2049                                 */
2050                                 prev_tsc = rte_rdtsc();
2051                                 do {
2052                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2053                                                 pkts_burst, MAX_PKT_BURST);
2054                                         for (i = 0; i < nb_rx; i++)
2055                                                 rte_pktmbuf_free(pkts_burst[i]);
2056
2057                                         cur_tsc = rte_rdtsc();
2058                                         diff_tsc = cur_tsc - prev_tsc;
2059                                         timer_tsc += diff_tsc;
2060                                 } while ((nb_rx > 0) &&
2061                                         (timer_tsc < timer_period));
2062                                 timer_tsc = 0;
2063                         }
2064                 }
2065                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2066         }
2067 }
2068
2069 static void
2070 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2071 {
2072         struct fwd_stream **fsm;
2073         streamid_t nb_fs;
2074         streamid_t sm_id;
2075 #ifdef RTE_LIBRTE_BITRATESTATS
2076         uint64_t tics_per_1sec;
2077         uint64_t tics_datum;
2078         uint64_t tics_current;
2079         uint16_t i, cnt_ports;
2080
2081         cnt_ports = nb_ports;
2082         tics_datum = rte_rdtsc();
2083         tics_per_1sec = rte_get_timer_hz();
2084 #endif
2085         fsm = &fwd_streams[fc->stream_idx];
2086         nb_fs = fc->stream_nb;
2087         do {
2088                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2089                         (*pkt_fwd)(fsm[sm_id]);
2090 #ifdef RTE_LIBRTE_BITRATESTATS
2091                 if (bitrate_enabled != 0 &&
2092                                 bitrate_lcore_id == rte_lcore_id()) {
2093                         tics_current = rte_rdtsc();
2094                         if (tics_current - tics_datum >= tics_per_1sec) {
2095                                 /* Periodic bitrate calculation */
2096                                 for (i = 0; i < cnt_ports; i++)
2097                                         rte_stats_bitrate_calc(bitrate_data,
2098                                                 ports_ids[i]);
2099                                 tics_datum = tics_current;
2100                         }
2101                 }
2102 #endif
2103 #ifdef RTE_LIBRTE_LATENCY_STATS
2104                 if (latencystats_enabled != 0 &&
2105                                 latencystats_lcore_id == rte_lcore_id())
2106                         rte_latencystats_update();
2107 #endif
2108
2109         } while (! fc->stopped);
2110 }
2111
2112 static int
2113 start_pkt_forward_on_core(void *fwd_arg)
2114 {
2115         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2116                              cur_fwd_config.fwd_eng->packet_fwd);
2117         return 0;
2118 }
2119
2120 /*
2121  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2122  * Used to start communication flows in network loopback test configurations.
2123  */
2124 static int
2125 run_one_txonly_burst_on_core(void *fwd_arg)
2126 {
2127         struct fwd_lcore *fwd_lc;
2128         struct fwd_lcore tmp_lcore;
2129
2130         fwd_lc = (struct fwd_lcore *) fwd_arg;
2131         tmp_lcore = *fwd_lc;
2132         tmp_lcore.stopped = 1;
2133         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2134         return 0;
2135 }
2136
2137 /*
2138  * Launch packet forwarding:
2139  *     - Setup per-port forwarding context.
2140  *     - launch logical cores with their forwarding configuration.
2141  */
2142 static void
2143 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2144 {
2145         port_fwd_begin_t port_fwd_begin;
2146         unsigned int i;
2147         unsigned int lc_id;
2148         int diag;
2149
2150         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2151         if (port_fwd_begin != NULL) {
2152                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2153                         (*port_fwd_begin)(fwd_ports_ids[i]);
2154         }
2155         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2156                 lc_id = fwd_lcores_cpuids[i];
2157                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2158                         fwd_lcores[i]->stopped = 0;
2159                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2160                                                      fwd_lcores[i], lc_id);
2161                         if (diag != 0)
2162                                 printf("launch lcore %u failed - diag=%d\n",
2163                                        lc_id, diag);
2164                 }
2165         }
2166 }
2167
2168 /*
2169  * Launch packet forwarding configuration.
2170  */
2171 void
2172 start_packet_forwarding(int with_tx_first)
2173 {
2174         port_fwd_begin_t port_fwd_begin;
2175         port_fwd_end_t  port_fwd_end;
2176         struct rte_port *port;
2177         unsigned int i;
2178         portid_t   pt_id;
2179
2180         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2181                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2182
2183         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2184                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2185
2186         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2187                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2188                 (!nb_rxq || !nb_txq))
2189                 rte_exit(EXIT_FAILURE,
2190                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2191                         cur_fwd_eng->fwd_mode_name);
2192
2193         if (all_ports_started() == 0) {
2194                 printf("Not all ports were started\n");
2195                 return;
2196         }
2197         if (test_done == 0) {
2198                 printf("Packet forwarding already started\n");
2199                 return;
2200         }
2201
2202
2203         if(dcb_test) {
2204                 for (i = 0; i < nb_fwd_ports; i++) {
2205                         pt_id = fwd_ports_ids[i];
2206                         port = &ports[pt_id];
2207                         if (!port->dcb_flag) {
2208                                 printf("In DCB mode, all forwarding ports must "
2209                                        "be configured in this mode.\n");
2210                                 return;
2211                         }
2212                 }
2213                 if (nb_fwd_lcores == 1) {
2214                         printf("In DCB mode,the nb forwarding cores "
2215                                "should be larger than 1.\n");
2216                         return;
2217                 }
2218         }
2219         test_done = 0;
2220
2221         fwd_config_setup();
2222
2223         if(!no_flush_rx)
2224                 flush_fwd_rx_queues();
2225
2226         pkt_fwd_config_display(&cur_fwd_config);
2227         rxtx_config_display();
2228
2229         fwd_stats_reset();
2230         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2231                 pt_id = fwd_ports_ids[i];
2232                 port = &ports[pt_id];
2233                 map_port_queue_stats_mapping_registers(pt_id, port);
2234         }
2235         if (with_tx_first) {
2236                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2237                 if (port_fwd_begin != NULL) {
2238                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2239                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2240                 }
2241                 while (with_tx_first--) {
2242                         launch_packet_forwarding(
2243                                         run_one_txonly_burst_on_core);
2244                         rte_eal_mp_wait_lcore();
2245                 }
2246                 port_fwd_end = tx_only_engine.port_fwd_end;
2247                 if (port_fwd_end != NULL) {
2248                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2249                                 (*port_fwd_end)(fwd_ports_ids[i]);
2250                 }
2251         }
2252         launch_packet_forwarding(start_pkt_forward_on_core);
2253 }
2254
2255 void
2256 stop_packet_forwarding(void)
2257 {
2258         port_fwd_end_t port_fwd_end;
2259         lcoreid_t lc_id;
2260         portid_t pt_id;
2261         int i;
2262
2263         if (test_done) {
2264                 printf("Packet forwarding not started\n");
2265                 return;
2266         }
2267         printf("Telling cores to stop...");
2268         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2269                 fwd_lcores[lc_id]->stopped = 1;
2270         printf("\nWaiting for lcores to finish...\n");
2271         rte_eal_mp_wait_lcore();
2272         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2273         if (port_fwd_end != NULL) {
2274                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2275                         pt_id = fwd_ports_ids[i];
2276                         (*port_fwd_end)(pt_id);
2277                 }
2278         }
2279
2280         fwd_stats_display();
2281
2282         printf("\nDone.\n");
2283         test_done = 1;
2284 }
2285
2286 void
2287 dev_set_link_up(portid_t pid)
2288 {
2289         if (rte_eth_dev_set_link_up(pid) < 0)
2290                 printf("\nSet link up fail.\n");
2291 }
2292
2293 void
2294 dev_set_link_down(portid_t pid)
2295 {
2296         if (rte_eth_dev_set_link_down(pid) < 0)
2297                 printf("\nSet link down fail.\n");
2298 }
2299
2300 static int
2301 all_ports_started(void)
2302 {
2303         portid_t pi;
2304         struct rte_port *port;
2305
2306         RTE_ETH_FOREACH_DEV(pi) {
2307                 port = &ports[pi];
2308                 /* Check if there is a port which is not started */
2309                 if ((port->port_status != RTE_PORT_STARTED) &&
2310                         (port->slave_flag == 0))
2311                         return 0;
2312         }
2313
2314         /* No port is not started */
2315         return 1;
2316 }
2317
2318 int
2319 port_is_stopped(portid_t port_id)
2320 {
2321         struct rte_port *port = &ports[port_id];
2322
2323         if ((port->port_status != RTE_PORT_STOPPED) &&
2324             (port->slave_flag == 0))
2325                 return 0;
2326         return 1;
2327 }
2328
2329 int
2330 all_ports_stopped(void)
2331 {
2332         portid_t pi;
2333
2334         RTE_ETH_FOREACH_DEV(pi) {
2335                 if (!port_is_stopped(pi))
2336                         return 0;
2337         }
2338
2339         return 1;
2340 }
2341
2342 int
2343 port_is_started(portid_t port_id)
2344 {
2345         if (port_id_is_invalid(port_id, ENABLED_WARN))
2346                 return 0;
2347
2348         if (ports[port_id].port_status != RTE_PORT_STARTED)
2349                 return 0;
2350
2351         return 1;
2352 }
2353
2354 /* Configure the Rx and Tx hairpin queues for the selected port. */
2355 static int
2356 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2357 {
2358         queueid_t qi;
2359         struct rte_eth_hairpin_conf hairpin_conf = {
2360                 .peer_count = 1,
2361         };
2362         int i;
2363         int diag;
2364         struct rte_port *port = &ports[pi];
2365         uint16_t peer_rx_port = pi;
2366         uint16_t peer_tx_port = pi;
2367         uint32_t manual = 1;
2368         uint32_t tx_exp = hairpin_mode & 0x10;
2369
2370         if (!(hairpin_mode & 0xf)) {
2371                 peer_rx_port = pi;
2372                 peer_tx_port = pi;
2373                 manual = 0;
2374         } else if (hairpin_mode & 0x1) {
2375                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2376                                                        RTE_ETH_DEV_NO_OWNER);
2377                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2378                         peer_tx_port = rte_eth_find_next_owned_by(0,
2379                                                 RTE_ETH_DEV_NO_OWNER);
2380                 if (p_pi != RTE_MAX_ETHPORTS) {
2381                         peer_rx_port = p_pi;
2382                 } else {
2383                         uint16_t next_pi;
2384
2385                         /* Last port will be the peer RX port of the first. */
2386                         RTE_ETH_FOREACH_DEV(next_pi)
2387                                 peer_rx_port = next_pi;
2388                 }
2389                 manual = 1;
2390         } else if (hairpin_mode & 0x2) {
2391                 if (cnt_pi & 0x1) {
2392                         peer_rx_port = p_pi;
2393                 } else {
2394                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2395                                                 RTE_ETH_DEV_NO_OWNER);
2396                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2397                                 peer_rx_port = pi;
2398                 }
2399                 peer_tx_port = peer_rx_port;
2400                 manual = 1;
2401         }
2402
2403         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2404                 hairpin_conf.peers[0].port = peer_rx_port;
2405                 hairpin_conf.peers[0].queue = i + nb_rxq;
2406                 hairpin_conf.manual_bind = !!manual;
2407                 hairpin_conf.tx_explicit = !!tx_exp;
2408                 diag = rte_eth_tx_hairpin_queue_setup
2409                         (pi, qi, nb_txd, &hairpin_conf);
2410                 i++;
2411                 if (diag == 0)
2412                         continue;
2413
2414                 /* Fail to setup rx queue, return */
2415                 if (rte_atomic16_cmpset(&(port->port_status),
2416                                         RTE_PORT_HANDLING,
2417                                         RTE_PORT_STOPPED) == 0)
2418                         printf("Port %d can not be set back "
2419                                         "to stopped\n", pi);
2420                 printf("Fail to configure port %d hairpin "
2421                                 "queues\n", pi);
2422                 /* try to reconfigure queues next time */
2423                 port->need_reconfig_queues = 1;
2424                 return -1;
2425         }
2426         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2427                 hairpin_conf.peers[0].port = peer_tx_port;
2428                 hairpin_conf.peers[0].queue = i + nb_txq;
2429                 hairpin_conf.manual_bind = !!manual;
2430                 hairpin_conf.tx_explicit = !!tx_exp;
2431                 diag = rte_eth_rx_hairpin_queue_setup
2432                         (pi, qi, nb_rxd, &hairpin_conf);
2433                 i++;
2434                 if (diag == 0)
2435                         continue;
2436
2437                 /* Fail to setup rx queue, return */
2438                 if (rte_atomic16_cmpset(&(port->port_status),
2439                                         RTE_PORT_HANDLING,
2440                                         RTE_PORT_STOPPED) == 0)
2441                         printf("Port %d can not be set back "
2442                                         "to stopped\n", pi);
2443                 printf("Fail to configure port %d hairpin "
2444                                 "queues\n", pi);
2445                 /* try to reconfigure queues next time */
2446                 port->need_reconfig_queues = 1;
2447                 return -1;
2448         }
2449         return 0;
2450 }
2451
2452 int
2453 start_port(portid_t pid)
2454 {
2455         int diag, need_check_link_status = -1;
2456         portid_t pi;
2457         portid_t p_pi = RTE_MAX_ETHPORTS;
2458         portid_t pl[RTE_MAX_ETHPORTS];
2459         portid_t peer_pl[RTE_MAX_ETHPORTS];
2460         uint16_t cnt_pi = 0;
2461         uint16_t cfg_pi = 0;
2462         int peer_pi;
2463         queueid_t qi;
2464         struct rte_port *port;
2465         struct rte_ether_addr mac_addr;
2466         struct rte_eth_hairpin_cap cap;
2467
2468         if (port_id_is_invalid(pid, ENABLED_WARN))
2469                 return 0;
2470
2471         if(dcb_config)
2472                 dcb_test = 1;
2473         RTE_ETH_FOREACH_DEV(pi) {
2474                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2475                         continue;
2476
2477                 need_check_link_status = 0;
2478                 port = &ports[pi];
2479                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2480                                                  RTE_PORT_HANDLING) == 0) {
2481                         printf("Port %d is now not stopped\n", pi);
2482                         continue;
2483                 }
2484
2485                 if (port->need_reconfig > 0) {
2486                         port->need_reconfig = 0;
2487
2488                         if (flow_isolate_all) {
2489                                 int ret = port_flow_isolate(pi, 1);
2490                                 if (ret) {
2491                                         printf("Failed to apply isolated"
2492                                                " mode on port %d\n", pi);
2493                                         return -1;
2494                                 }
2495                         }
2496                         configure_rxtx_dump_callbacks(0);
2497                         printf("Configuring Port %d (socket %u)\n", pi,
2498                                         port->socket_id);
2499                         if (nb_hairpinq > 0 &&
2500                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2501                                 printf("Port %d doesn't support hairpin "
2502                                        "queues\n", pi);
2503                                 return -1;
2504                         }
2505                         /* configure port */
2506                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2507                                                      nb_txq + nb_hairpinq,
2508                                                      &(port->dev_conf));
2509                         if (diag != 0) {
2510                                 if (rte_atomic16_cmpset(&(port->port_status),
2511                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2512                                         printf("Port %d can not be set back "
2513                                                         "to stopped\n", pi);
2514                                 printf("Fail to configure port %d\n", pi);
2515                                 /* try to reconfigure port next time */
2516                                 port->need_reconfig = 1;
2517                                 return -1;
2518                         }
2519                 }
2520                 if (port->need_reconfig_queues > 0) {
2521                         port->need_reconfig_queues = 0;
2522                         /* setup tx queues */
2523                         for (qi = 0; qi < nb_txq; qi++) {
2524                                 if ((numa_support) &&
2525                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2526                                         diag = rte_eth_tx_queue_setup(pi, qi,
2527                                                 port->nb_tx_desc[qi],
2528                                                 txring_numa[pi],
2529                                                 &(port->tx_conf[qi]));
2530                                 else
2531                                         diag = rte_eth_tx_queue_setup(pi, qi,
2532                                                 port->nb_tx_desc[qi],
2533                                                 port->socket_id,
2534                                                 &(port->tx_conf[qi]));
2535
2536                                 if (diag == 0)
2537                                         continue;
2538
2539                                 /* Fail to setup tx queue, return */
2540                                 if (rte_atomic16_cmpset(&(port->port_status),
2541                                                         RTE_PORT_HANDLING,
2542                                                         RTE_PORT_STOPPED) == 0)
2543                                         printf("Port %d can not be set back "
2544                                                         "to stopped\n", pi);
2545                                 printf("Fail to configure port %d tx queues\n",
2546                                        pi);
2547                                 /* try to reconfigure queues next time */
2548                                 port->need_reconfig_queues = 1;
2549                                 return -1;
2550                         }
2551                         for (qi = 0; qi < nb_rxq; qi++) {
2552                                 /* setup rx queues */
2553                                 if ((numa_support) &&
2554                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2555                                         struct rte_mempool * mp =
2556                                                 mbuf_pool_find
2557                                                         (rxring_numa[pi], 0);
2558                                         if (mp == NULL) {
2559                                                 printf("Failed to setup RX queue:"
2560                                                         "No mempool allocation"
2561                                                         " on the socket %d\n",
2562                                                         rxring_numa[pi]);
2563                                                 return -1;
2564                                         }
2565
2566                                         diag = rte_eth_rx_queue_setup(pi, qi,
2567                                              port->nb_rx_desc[qi],
2568                                              rxring_numa[pi],
2569                                              &(port->rx_conf[qi]),
2570                                              mp);
2571                                 } else {
2572                                         struct rte_mempool *mp =
2573                                                 mbuf_pool_find
2574                                                         (port->socket_id, 0);
2575                                         if (mp == NULL) {
2576                                                 printf("Failed to setup RX queue:"
2577                                                         "No mempool allocation"
2578                                                         " on the socket %d\n",
2579                                                         port->socket_id);
2580                                                 return -1;
2581                                         }
2582                                         diag = rte_eth_rx_queue_setup(pi, qi,
2583                                              port->nb_rx_desc[qi],
2584                                              port->socket_id,
2585                                              &(port->rx_conf[qi]),
2586                                              mp);
2587                                 }
2588                                 if (diag == 0)
2589                                         continue;
2590
2591                                 /* Fail to setup rx queue, return */
2592                                 if (rte_atomic16_cmpset(&(port->port_status),
2593                                                         RTE_PORT_HANDLING,
2594                                                         RTE_PORT_STOPPED) == 0)
2595                                         printf("Port %d can not be set back "
2596                                                         "to stopped\n", pi);
2597                                 printf("Fail to configure port %d rx queues\n",
2598                                        pi);
2599                                 /* try to reconfigure queues next time */
2600                                 port->need_reconfig_queues = 1;
2601                                 return -1;
2602                         }
2603                         /* setup hairpin queues */
2604                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2605                                 return -1;
2606                 }
2607                 configure_rxtx_dump_callbacks(verbose_level);
2608                 if (clear_ptypes) {
2609                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2610                                         NULL, 0);
2611                         if (diag < 0)
2612                                 printf(
2613                                 "Port %d: Failed to disable Ptype parsing\n",
2614                                 pi);
2615                 }
2616
2617                 p_pi = pi;
2618                 cnt_pi++;
2619
2620                 /* start port */
2621                 if (rte_eth_dev_start(pi) < 0) {
2622                         printf("Fail to start port %d\n", pi);
2623
2624                         /* Fail to setup rx queue, return */
2625                         if (rte_atomic16_cmpset(&(port->port_status),
2626                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2627                                 printf("Port %d can not be set back to "
2628                                                         "stopped\n", pi);
2629                         continue;
2630                 }
2631
2632                 if (rte_atomic16_cmpset(&(port->port_status),
2633                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2634                         printf("Port %d can not be set into started\n", pi);
2635
2636                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2637                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2638                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2639                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2640                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2641
2642                 /* at least one port started, need checking link status */
2643                 need_check_link_status = 1;
2644
2645                 pl[cfg_pi++] = pi;
2646         }
2647
2648         if (need_check_link_status == 1 && !no_link_check)
2649                 check_all_ports_link_status(RTE_PORT_ALL);
2650         else if (need_check_link_status == 0)
2651                 printf("Please stop the ports first\n");
2652
2653         if (hairpin_mode & 0xf) {
2654                 uint16_t i;
2655                 int j;
2656
2657                 /* bind all started hairpin ports */
2658                 for (i = 0; i < cfg_pi; i++) {
2659                         pi = pl[i];
2660                         /* bind current Tx to all peer Rx */
2661                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2662                                                         RTE_MAX_ETHPORTS, 1);
2663                         if (peer_pi < 0)
2664                                 return peer_pi;
2665                         for (j = 0; j < peer_pi; j++) {
2666                                 if (!port_is_started(peer_pl[j]))
2667                                         continue;
2668                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2669                                 if (diag < 0) {
2670                                         printf("Error during binding hairpin"
2671                                                " Tx port %u to %u: %s\n",
2672                                                pi, peer_pl[j],
2673                                                rte_strerror(-diag));
2674                                         return -1;
2675                                 }
2676                         }
2677                         /* bind all peer Tx to current Rx */
2678                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2679                                                         RTE_MAX_ETHPORTS, 0);
2680                         if (peer_pi < 0)
2681                                 return peer_pi;
2682                         for (j = 0; j < peer_pi; j++) {
2683                                 if (!port_is_started(peer_pl[j]))
2684                                         continue;
2685                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2686                                 if (diag < 0) {
2687                                         printf("Error during binding hairpin"
2688                                                " Tx port %u to %u: %s\n",
2689                                                peer_pl[j], pi,
2690                                                rte_strerror(-diag));
2691                                         return -1;
2692                                 }
2693                         }
2694                 }
2695         }
2696
2697         printf("Done\n");
2698         return 0;
2699 }
2700
2701 void
2702 stop_port(portid_t pid)
2703 {
2704         portid_t pi;
2705         struct rte_port *port;
2706         int need_check_link_status = 0;
2707         portid_t peer_pl[RTE_MAX_ETHPORTS];
2708         int peer_pi;
2709
2710         if (dcb_test) {
2711                 dcb_test = 0;
2712                 dcb_config = 0;
2713         }
2714
2715         if (port_id_is_invalid(pid, ENABLED_WARN))
2716                 return;
2717
2718         printf("Stopping ports...\n");
2719
2720         RTE_ETH_FOREACH_DEV(pi) {
2721                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2722                         continue;
2723
2724                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2725                         printf("Please remove port %d from forwarding configuration.\n", pi);
2726                         continue;
2727                 }
2728
2729                 if (port_is_bonding_slave(pi)) {
2730                         printf("Please remove port %d from bonded device.\n", pi);
2731                         continue;
2732                 }
2733
2734                 port = &ports[pi];
2735                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2736                                                 RTE_PORT_HANDLING) == 0)
2737                         continue;
2738
2739                 if (hairpin_mode & 0xf) {
2740                         int j;
2741
2742                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2743                         /* unbind all peer Tx from current Rx */
2744                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2745                                                         RTE_MAX_ETHPORTS, 0);
2746                         if (peer_pi < 0)
2747                                 continue;
2748                         for (j = 0; j < peer_pi; j++) {
2749                                 if (!port_is_started(peer_pl[j]))
2750                                         continue;
2751                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2752                         }
2753                 }
2754
2755                 rte_eth_dev_stop(pi);
2756
2757                 if (rte_atomic16_cmpset(&(port->port_status),
2758                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2759                         printf("Port %d can not be set into stopped\n", pi);
2760                 need_check_link_status = 1;
2761         }
2762         if (need_check_link_status && !no_link_check)
2763                 check_all_ports_link_status(RTE_PORT_ALL);
2764
2765         printf("Done\n");
2766 }
2767
2768 static void
2769 remove_invalid_ports_in(portid_t *array, portid_t *total)
2770 {
2771         portid_t i;
2772         portid_t new_total = 0;
2773
2774         for (i = 0; i < *total; i++)
2775                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2776                         array[new_total] = array[i];
2777                         new_total++;
2778                 }
2779         *total = new_total;
2780 }
2781
2782 static void
2783 remove_invalid_ports(void)
2784 {
2785         remove_invalid_ports_in(ports_ids, &nb_ports);
2786         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2787         nb_cfg_ports = nb_fwd_ports;
2788 }
2789
2790 void
2791 close_port(portid_t pid)
2792 {
2793         portid_t pi;
2794         struct rte_port *port;
2795
2796         if (port_id_is_invalid(pid, ENABLED_WARN))
2797                 return;
2798
2799         printf("Closing ports...\n");
2800
2801         RTE_ETH_FOREACH_DEV(pi) {
2802                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2803                         continue;
2804
2805                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2806                         printf("Please remove port %d from forwarding configuration.\n", pi);
2807                         continue;
2808                 }
2809
2810                 if (port_is_bonding_slave(pi)) {
2811                         printf("Please remove port %d from bonded device.\n", pi);
2812                         continue;
2813                 }
2814
2815                 port = &ports[pi];
2816                 if (rte_atomic16_cmpset(&(port->port_status),
2817                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2818                         printf("Port %d is already closed\n", pi);
2819                         continue;
2820                 }
2821
2822                 port_flow_flush(pi);
2823                 rte_eth_dev_close(pi);
2824         }
2825
2826         remove_invalid_ports();
2827         printf("Done\n");
2828 }
2829
2830 void
2831 reset_port(portid_t pid)
2832 {
2833         int diag;
2834         portid_t pi;
2835         struct rte_port *port;
2836
2837         if (port_id_is_invalid(pid, ENABLED_WARN))
2838                 return;
2839
2840         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2841                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2842                 printf("Can not reset port(s), please stop port(s) first.\n");
2843                 return;
2844         }
2845
2846         printf("Resetting ports...\n");
2847
2848         RTE_ETH_FOREACH_DEV(pi) {
2849                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2850                         continue;
2851
2852                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2853                         printf("Please remove port %d from forwarding "
2854                                "configuration.\n", pi);
2855                         continue;
2856                 }
2857
2858                 if (port_is_bonding_slave(pi)) {
2859                         printf("Please remove port %d from bonded device.\n",
2860                                pi);
2861                         continue;
2862                 }
2863
2864                 diag = rte_eth_dev_reset(pi);
2865                 if (diag == 0) {
2866                         port = &ports[pi];
2867                         port->need_reconfig = 1;
2868                         port->need_reconfig_queues = 1;
2869                 } else {
2870                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2871                 }
2872         }
2873
2874         printf("Done\n");
2875 }
2876
2877 void
2878 attach_port(char *identifier)
2879 {
2880         portid_t pi;
2881         struct rte_dev_iterator iterator;
2882
2883         printf("Attaching a new port...\n");
2884
2885         if (identifier == NULL) {
2886                 printf("Invalid parameters are specified\n");
2887                 return;
2888         }
2889
2890         if (rte_dev_probe(identifier) < 0) {
2891                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2892                 return;
2893         }
2894
2895         /* first attach mode: event */
2896         if (setup_on_probe_event) {
2897                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2898                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2899                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2900                                         ports[pi].need_setup != 0)
2901                                 setup_attached_port(pi);
2902                 return;
2903         }
2904
2905         /* second attach mode: iterator */
2906         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2907                 /* setup ports matching the devargs used for probing */
2908                 if (port_is_forwarding(pi))
2909                         continue; /* port was already attached before */
2910                 setup_attached_port(pi);
2911         }
2912 }
2913
2914 static void
2915 setup_attached_port(portid_t pi)
2916 {
2917         unsigned int socket_id;
2918         int ret;
2919
2920         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2921         /* if socket_id is invalid, set to the first available socket. */
2922         if (check_socket_id(socket_id) < 0)
2923                 socket_id = socket_ids[0];
2924         reconfig(pi, socket_id);
2925         ret = rte_eth_promiscuous_enable(pi);
2926         if (ret != 0)
2927                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2928                         pi, rte_strerror(-ret));
2929
2930         ports_ids[nb_ports++] = pi;
2931         fwd_ports_ids[nb_fwd_ports++] = pi;
2932         nb_cfg_ports = nb_fwd_ports;
2933         ports[pi].need_setup = 0;
2934         ports[pi].port_status = RTE_PORT_STOPPED;
2935
2936         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2937         printf("Done\n");
2938 }
2939
2940 static void
2941 detach_device(struct rte_device *dev)
2942 {
2943         portid_t sibling;
2944
2945         if (dev == NULL) {
2946                 printf("Device already removed\n");
2947                 return;
2948         }
2949
2950         printf("Removing a device...\n");
2951
2952         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2953                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2954                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2955                                 printf("Port %u not stopped\n", sibling);
2956                                 return;
2957                         }
2958                         port_flow_flush(sibling);
2959                 }
2960         }
2961
2962         if (rte_dev_remove(dev) < 0) {
2963                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2964                 return;
2965         }
2966         remove_invalid_ports();
2967
2968         printf("Device is detached\n");
2969         printf("Now total ports is %d\n", nb_ports);
2970         printf("Done\n");
2971         return;
2972 }
2973
2974 void
2975 detach_port_device(portid_t port_id)
2976 {
2977         if (port_id_is_invalid(port_id, ENABLED_WARN))
2978                 return;
2979
2980         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2981                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2982                         printf("Port not stopped\n");
2983                         return;
2984                 }
2985                 printf("Port was not closed\n");
2986         }
2987
2988         detach_device(rte_eth_devices[port_id].device);
2989 }
2990
2991 void
2992 detach_devargs(char *identifier)
2993 {
2994         struct rte_dev_iterator iterator;
2995         struct rte_devargs da;
2996         portid_t port_id;
2997
2998         printf("Removing a device...\n");
2999
3000         memset(&da, 0, sizeof(da));
3001         if (rte_devargs_parsef(&da, "%s", identifier)) {
3002                 printf("cannot parse identifier\n");
3003                 if (da.args)
3004                         free(da.args);
3005                 return;
3006         }
3007
3008         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3009                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3010                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3011                                 printf("Port %u not stopped\n", port_id);
3012                                 rte_eth_iterator_cleanup(&iterator);
3013                                 return;
3014                         }
3015                         port_flow_flush(port_id);
3016                 }
3017         }
3018
3019         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3020                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3021                             da.name, da.bus->name);
3022                 return;
3023         }
3024
3025         remove_invalid_ports();
3026
3027         printf("Device %s is detached\n", identifier);
3028         printf("Now total ports is %d\n", nb_ports);
3029         printf("Done\n");
3030 }
3031
3032 void
3033 pmd_test_exit(void)
3034 {
3035         portid_t pt_id;
3036         unsigned int i;
3037         int ret;
3038
3039         if (test_done == 0)
3040                 stop_packet_forwarding();
3041
3042         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3043                 if (mempools[i]) {
3044                         if (mp_alloc_type == MP_ALLOC_ANON)
3045                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3046                                                      NULL);
3047                 }
3048         }
3049         if (ports != NULL) {
3050                 no_link_check = 1;
3051                 RTE_ETH_FOREACH_DEV(pt_id) {
3052                         printf("\nStopping port %d...\n", pt_id);
3053                         fflush(stdout);
3054                         stop_port(pt_id);
3055                 }
3056                 RTE_ETH_FOREACH_DEV(pt_id) {
3057                         printf("\nShutting down port %d...\n", pt_id);
3058                         fflush(stdout);
3059                         close_port(pt_id);
3060                 }
3061         }
3062
3063         if (hot_plug) {
3064                 ret = rte_dev_event_monitor_stop();
3065                 if (ret) {
3066                         RTE_LOG(ERR, EAL,
3067                                 "fail to stop device event monitor.");
3068                         return;
3069                 }
3070
3071                 ret = rte_dev_event_callback_unregister(NULL,
3072                         dev_event_callback, NULL);
3073                 if (ret < 0) {
3074                         RTE_LOG(ERR, EAL,
3075                                 "fail to unregister device event callback.\n");
3076                         return;
3077                 }
3078
3079                 ret = rte_dev_hotplug_handle_disable();
3080                 if (ret) {
3081                         RTE_LOG(ERR, EAL,
3082                                 "fail to disable hotplug handling.\n");
3083                         return;
3084                 }
3085         }
3086         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3087                 if (mempools[i])
3088                         rte_mempool_free(mempools[i]);
3089         }
3090
3091         printf("\nBye...\n");
3092 }
3093
3094 typedef void (*cmd_func_t)(void);
3095 struct pmd_test_command {
3096         const char *cmd_name;
3097         cmd_func_t cmd_func;
3098 };
3099
3100 /* Check the link status of all ports in up to 9s, and print them finally */
3101 static void
3102 check_all_ports_link_status(uint32_t port_mask)
3103 {
3104 #define CHECK_INTERVAL 100 /* 100ms */
3105 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3106         portid_t portid;
3107         uint8_t count, all_ports_up, print_flag = 0;
3108         struct rte_eth_link link;
3109         int ret;
3110         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3111
3112         printf("Checking link statuses...\n");
3113         fflush(stdout);
3114         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3115                 all_ports_up = 1;
3116                 RTE_ETH_FOREACH_DEV(portid) {
3117                         if ((port_mask & (1 << portid)) == 0)
3118                                 continue;
3119                         memset(&link, 0, sizeof(link));
3120                         ret = rte_eth_link_get_nowait(portid, &link);
3121                         if (ret < 0) {
3122                                 all_ports_up = 0;
3123                                 if (print_flag == 1)
3124                                         printf("Port %u link get failed: %s\n",
3125                                                 portid, rte_strerror(-ret));
3126                                 continue;
3127                         }
3128                         /* print link status if flag set */
3129                         if (print_flag == 1) {
3130                                 rte_eth_link_to_str(link_status,
3131                                         sizeof(link_status), &link);
3132                                 printf("Port %d %s\n", portid, link_status);
3133                                 continue;
3134                         }
3135                         /* clear all_ports_up flag if any link down */
3136                         if (link.link_status == ETH_LINK_DOWN) {
3137                                 all_ports_up = 0;
3138                                 break;
3139                         }
3140                 }
3141                 /* after finally printing all link status, get out */
3142                 if (print_flag == 1)
3143                         break;
3144
3145                 if (all_ports_up == 0) {
3146                         fflush(stdout);
3147                         rte_delay_ms(CHECK_INTERVAL);
3148                 }
3149
3150                 /* set the print_flag if all ports up or timeout */
3151                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3152                         print_flag = 1;
3153                 }
3154
3155                 if (lsc_interrupt)
3156                         break;
3157         }
3158 }
3159
3160 static void
3161 rmv_port_callback(void *arg)
3162 {
3163         int need_to_start = 0;
3164         int org_no_link_check = no_link_check;
3165         portid_t port_id = (intptr_t)arg;
3166         struct rte_device *dev;
3167
3168         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3169
3170         if (!test_done && port_is_forwarding(port_id)) {
3171                 need_to_start = 1;
3172                 stop_packet_forwarding();
3173         }
3174         no_link_check = 1;
3175         stop_port(port_id);
3176         no_link_check = org_no_link_check;
3177
3178         /* Save rte_device pointer before closing ethdev port */
3179         dev = rte_eth_devices[port_id].device;
3180         close_port(port_id);
3181         detach_device(dev); /* might be already removed or have more ports */
3182
3183         if (need_to_start)
3184                 start_packet_forwarding(0);
3185 }
3186
3187 /* This function is used by the interrupt thread */
3188 static int
3189 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3190                   void *ret_param)
3191 {
3192         RTE_SET_USED(param);
3193         RTE_SET_USED(ret_param);
3194
3195         if (type >= RTE_ETH_EVENT_MAX) {
3196                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3197                         port_id, __func__, type);
3198                 fflush(stderr);
3199         } else if (event_print_mask & (UINT32_C(1) << type)) {
3200                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3201                         eth_event_desc[type]);
3202                 fflush(stdout);
3203         }
3204
3205         switch (type) {
3206         case RTE_ETH_EVENT_NEW:
3207                 ports[port_id].need_setup = 1;
3208                 ports[port_id].port_status = RTE_PORT_HANDLING;
3209                 break;
3210         case RTE_ETH_EVENT_INTR_RMV:
3211                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3212                         break;
3213                 if (rte_eal_alarm_set(100000,
3214                                 rmv_port_callback, (void *)(intptr_t)port_id))
3215                         fprintf(stderr, "Could not set up deferred device removal\n");
3216                 break;
3217         case RTE_ETH_EVENT_DESTROY:
3218                 ports[port_id].port_status = RTE_PORT_CLOSED;
3219                 printf("Port %u is closed\n", port_id);
3220                 break;
3221         default:
3222                 break;
3223         }
3224         return 0;
3225 }
3226
3227 static int
3228 register_eth_event_callback(void)
3229 {
3230         int ret;
3231         enum rte_eth_event_type event;
3232
3233         for (event = RTE_ETH_EVENT_UNKNOWN;
3234                         event < RTE_ETH_EVENT_MAX; event++) {
3235                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3236                                 event,
3237                                 eth_event_callback,
3238                                 NULL);
3239                 if (ret != 0) {
3240                         TESTPMD_LOG(ERR, "Failed to register callback for "
3241                                         "%s event\n", eth_event_desc[event]);
3242                         return -1;
3243                 }
3244         }
3245
3246         return 0;
3247 }
3248
3249 /* This function is used by the interrupt thread */
3250 static void
3251 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3252                              __rte_unused void *arg)
3253 {
3254         uint16_t port_id;
3255         int ret;
3256
3257         if (type >= RTE_DEV_EVENT_MAX) {
3258                 fprintf(stderr, "%s called upon invalid event %d\n",
3259                         __func__, type);
3260                 fflush(stderr);
3261         }
3262
3263         switch (type) {
3264         case RTE_DEV_EVENT_REMOVE:
3265                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3266                         device_name);
3267                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3268                 if (ret) {
3269                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3270                                 device_name);
3271                         return;
3272                 }
3273                 /*
3274                  * Because the user's callback is invoked in eal interrupt
3275                  * callback, the interrupt callback need to be finished before
3276                  * it can be unregistered when detaching device. So finish
3277                  * callback soon and use a deferred removal to detach device
3278                  * is need. It is a workaround, once the device detaching be
3279                  * moved into the eal in the future, the deferred removal could
3280                  * be deleted.
3281                  */
3282                 if (rte_eal_alarm_set(100000,
3283                                 rmv_port_callback, (void *)(intptr_t)port_id))
3284                         RTE_LOG(ERR, EAL,
3285                                 "Could not set up deferred device removal\n");
3286                 break;
3287         case RTE_DEV_EVENT_ADD:
3288                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3289                         device_name);
3290                 /* TODO: After finish kernel driver binding,
3291                  * begin to attach port.
3292                  */
3293                 break;
3294         default:
3295                 break;
3296         }
3297 }
3298
3299 static int
3300 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3301 {
3302         uint16_t i;
3303         int diag;
3304         uint8_t mapping_found = 0;
3305
3306         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3307                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3308                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3309                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3310                                         tx_queue_stats_mappings[i].queue_id,
3311                                         tx_queue_stats_mappings[i].stats_counter_id);
3312                         if (diag != 0)
3313                                 return diag;
3314                         mapping_found = 1;
3315                 }
3316         }
3317         if (mapping_found)
3318                 port->tx_queue_stats_mapping_enabled = 1;
3319         return 0;
3320 }
3321
3322 static int
3323 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3324 {
3325         uint16_t i;
3326         int diag;
3327         uint8_t mapping_found = 0;
3328
3329         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3330                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3331                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3332                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3333                                         rx_queue_stats_mappings[i].queue_id,
3334                                         rx_queue_stats_mappings[i].stats_counter_id);
3335                         if (diag != 0)
3336                                 return diag;
3337                         mapping_found = 1;
3338                 }
3339         }
3340         if (mapping_found)
3341                 port->rx_queue_stats_mapping_enabled = 1;
3342         return 0;
3343 }
3344
3345 static void
3346 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3347 {
3348         int diag = 0;
3349
3350         diag = set_tx_queue_stats_mapping_registers(pi, port);
3351         if (diag != 0) {
3352                 if (diag == -ENOTSUP) {
3353                         port->tx_queue_stats_mapping_enabled = 0;
3354                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3355                 }
3356                 else
3357                         rte_exit(EXIT_FAILURE,
3358                                         "set_tx_queue_stats_mapping_registers "
3359                                         "failed for port id=%d diag=%d\n",
3360                                         pi, diag);
3361         }
3362
3363         diag = set_rx_queue_stats_mapping_registers(pi, port);
3364         if (diag != 0) {
3365                 if (diag == -ENOTSUP) {
3366                         port->rx_queue_stats_mapping_enabled = 0;
3367                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3368                 }
3369                 else
3370                         rte_exit(EXIT_FAILURE,
3371                                         "set_rx_queue_stats_mapping_registers "
3372                                         "failed for port id=%d diag=%d\n",
3373                                         pi, diag);
3374         }
3375 }
3376
3377 static void
3378 rxtx_port_config(struct rte_port *port)
3379 {
3380         uint16_t qid;
3381         uint64_t offloads;
3382
3383         for (qid = 0; qid < nb_rxq; qid++) {
3384                 offloads = port->rx_conf[qid].offloads;
3385                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3386                 if (offloads != 0)
3387                         port->rx_conf[qid].offloads = offloads;
3388
3389                 /* Check if any Rx parameters have been passed */
3390                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3391                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3392
3393                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3394                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3395
3396                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3397                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3398
3399                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3400                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3401
3402                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3403                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3404
3405                 port->nb_rx_desc[qid] = nb_rxd;
3406         }
3407
3408         for (qid = 0; qid < nb_txq; qid++) {
3409                 offloads = port->tx_conf[qid].offloads;
3410                 port->tx_conf[qid] = port->dev_info.default_txconf;
3411                 if (offloads != 0)
3412                         port->tx_conf[qid].offloads = offloads;
3413
3414                 /* Check if any Tx parameters have been passed */
3415                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3416                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3417
3418                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3419                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3420
3421                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3422                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3423
3424                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3425                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3426
3427                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3428                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3429
3430                 port->nb_tx_desc[qid] = nb_txd;
3431         }
3432 }
3433
3434 void
3435 init_port_config(void)
3436 {
3437         portid_t pid;
3438         struct rte_port *port;
3439         int ret;
3440
3441         RTE_ETH_FOREACH_DEV(pid) {
3442                 port = &ports[pid];
3443                 port->dev_conf.fdir_conf = fdir_conf;
3444
3445                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3446                 if (ret != 0)
3447                         return;
3448
3449                 if (nb_rxq > 1) {
3450                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3451                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3452                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3453                 } else {
3454                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3455                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3456                 }
3457
3458                 if (port->dcb_flag == 0) {
3459                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3460                                 port->dev_conf.rxmode.mq_mode =
3461                                         (enum rte_eth_rx_mq_mode)
3462                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3463                         else
3464                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3465                 }
3466
3467                 rxtx_port_config(port);
3468
3469                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3470                 if (ret != 0)
3471                         return;
3472
3473                 map_port_queue_stats_mapping_registers(pid, port);
3474 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3475                 rte_pmd_ixgbe_bypass_init(pid);
3476 #endif
3477
3478                 if (lsc_interrupt &&
3479                     (rte_eth_devices[pid].data->dev_flags &
3480                      RTE_ETH_DEV_INTR_LSC))
3481                         port->dev_conf.intr_conf.lsc = 1;
3482                 if (rmv_interrupt &&
3483                     (rte_eth_devices[pid].data->dev_flags &
3484                      RTE_ETH_DEV_INTR_RMV))
3485                         port->dev_conf.intr_conf.rmv = 1;
3486         }
3487 }
3488
3489 void set_port_slave_flag(portid_t slave_pid)
3490 {
3491         struct rte_port *port;
3492
3493         port = &ports[slave_pid];
3494         port->slave_flag = 1;
3495 }
3496
3497 void clear_port_slave_flag(portid_t slave_pid)
3498 {
3499         struct rte_port *port;
3500
3501         port = &ports[slave_pid];
3502         port->slave_flag = 0;
3503 }
3504
3505 uint8_t port_is_bonding_slave(portid_t slave_pid)
3506 {
3507         struct rte_port *port;
3508
3509         port = &ports[slave_pid];
3510         if ((rte_eth_devices[slave_pid].data->dev_flags &
3511             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3512                 return 1;
3513         return 0;
3514 }
3515
3516 const uint16_t vlan_tags[] = {
3517                 0,  1,  2,  3,  4,  5,  6,  7,
3518                 8,  9, 10, 11,  12, 13, 14, 15,
3519                 16, 17, 18, 19, 20, 21, 22, 23,
3520                 24, 25, 26, 27, 28, 29, 30, 31
3521 };
3522
3523 static  int
3524 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3525                  enum dcb_mode_enable dcb_mode,
3526                  enum rte_eth_nb_tcs num_tcs,
3527                  uint8_t pfc_en)
3528 {
3529         uint8_t i;
3530         int32_t rc;
3531         struct rte_eth_rss_conf rss_conf;
3532
3533         /*
3534          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3535          * given above, and the number of traffic classes available for use.
3536          */
3537         if (dcb_mode == DCB_VT_ENABLED) {
3538                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3539                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3540                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3541                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3542
3543                 /* VMDQ+DCB RX and TX configurations */
3544                 vmdq_rx_conf->enable_default_pool = 0;
3545                 vmdq_rx_conf->default_pool = 0;
3546                 vmdq_rx_conf->nb_queue_pools =
3547                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3548                 vmdq_tx_conf->nb_queue_pools =
3549                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3550
3551                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3552                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3553                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3554                         vmdq_rx_conf->pool_map[i].pools =
3555                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3556                 }
3557                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3558                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3559                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3560                 }
3561
3562                 /* set DCB mode of RX and TX of multiple queues */
3563                 eth_conf->rxmode.mq_mode =
3564                                 (enum rte_eth_rx_mq_mode)
3565                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3566                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3567         } else {
3568                 struct rte_eth_dcb_rx_conf *rx_conf =
3569                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3570                 struct rte_eth_dcb_tx_conf *tx_conf =
3571                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3572
3573                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3574
3575                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3576                 if (rc != 0)
3577                         return rc;
3578
3579                 rx_conf->nb_tcs = num_tcs;
3580                 tx_conf->nb_tcs = num_tcs;
3581
3582                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3583                         rx_conf->dcb_tc[i] = i % num_tcs;
3584                         tx_conf->dcb_tc[i] = i % num_tcs;
3585                 }
3586
3587                 eth_conf->rxmode.mq_mode =
3588                                 (enum rte_eth_rx_mq_mode)
3589                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3590                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3591                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3592         }
3593
3594         if (pfc_en)
3595                 eth_conf->dcb_capability_en =
3596                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3597         else
3598                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3599
3600         return 0;
3601 }
3602
3603 int
3604 init_port_dcb_config(portid_t pid,
3605                      enum dcb_mode_enable dcb_mode,
3606                      enum rte_eth_nb_tcs num_tcs,
3607                      uint8_t pfc_en)
3608 {
3609         struct rte_eth_conf port_conf;
3610         struct rte_port *rte_port;
3611         int retval;
3612         uint16_t i;
3613
3614         rte_port = &ports[pid];
3615
3616         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3617         /* Enter DCB configuration status */
3618         dcb_config = 1;
3619
3620         port_conf.rxmode = rte_port->dev_conf.rxmode;
3621         port_conf.txmode = rte_port->dev_conf.txmode;
3622
3623         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3624         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3625         if (retval < 0)
3626                 return retval;
3627         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3628
3629         /* re-configure the device . */
3630         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3631         if (retval < 0)
3632                 return retval;
3633
3634         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3635         if (retval != 0)
3636                 return retval;
3637
3638         /* If dev_info.vmdq_pool_base is greater than 0,
3639          * the queue id of vmdq pools is started after pf queues.
3640          */
3641         if (dcb_mode == DCB_VT_ENABLED &&
3642             rte_port->dev_info.vmdq_pool_base > 0) {
3643                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3644                         " for port %d.", pid);
3645                 return -1;
3646         }
3647
3648         /* Assume the ports in testpmd have the same dcb capability
3649          * and has the same number of rxq and txq in dcb mode
3650          */
3651         if (dcb_mode == DCB_VT_ENABLED) {
3652                 if (rte_port->dev_info.max_vfs > 0) {
3653                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3654                         nb_txq = rte_port->dev_info.nb_tx_queues;
3655                 } else {
3656                         nb_rxq = rte_port->dev_info.max_rx_queues;
3657                         nb_txq = rte_port->dev_info.max_tx_queues;
3658                 }
3659         } else {
3660                 /*if vt is disabled, use all pf queues */
3661                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3662                         nb_rxq = rte_port->dev_info.max_rx_queues;
3663                         nb_txq = rte_port->dev_info.max_tx_queues;
3664                 } else {
3665                         nb_rxq = (queueid_t)num_tcs;
3666                         nb_txq = (queueid_t)num_tcs;
3667
3668                 }
3669         }
3670         rx_free_thresh = 64;
3671
3672         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3673
3674         rxtx_port_config(rte_port);
3675         /* VLAN filter */
3676         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3677         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3678                 rx_vft_set(pid, vlan_tags[i], 1);
3679
3680         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3681         if (retval != 0)
3682                 return retval;
3683
3684         map_port_queue_stats_mapping_registers(pid, rte_port);
3685
3686         rte_port->dcb_flag = 1;
3687
3688         return 0;
3689 }
3690
3691 static void
3692 init_port(void)
3693 {
3694         int i;
3695
3696         /* Configuration of Ethernet ports. */
3697         ports = rte_zmalloc("testpmd: ports",
3698                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3699                             RTE_CACHE_LINE_SIZE);
3700         if (ports == NULL) {
3701                 rte_exit(EXIT_FAILURE,
3702                                 "rte_zmalloc(%d struct rte_port) failed\n",
3703                                 RTE_MAX_ETHPORTS);
3704         }
3705         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3706                 LIST_INIT(&ports[i].flow_tunnel_list);
3707         /* Initialize ports NUMA structures */
3708         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3709         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3710         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3711 }
3712
3713 static void
3714 force_quit(void)
3715 {
3716         pmd_test_exit();
3717         prompt_exit();
3718 }
3719
3720 static void
3721 print_stats(void)
3722 {
3723         uint8_t i;
3724         const char clr[] = { 27, '[', '2', 'J', '\0' };
3725         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3726
3727         /* Clear screen and move to top left */
3728         printf("%s%s", clr, top_left);
3729
3730         printf("\nPort statistics ====================================");
3731         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3732                 nic_stats_display(fwd_ports_ids[i]);
3733
3734         fflush(stdout);
3735 }
3736
3737 static void
3738 signal_handler(int signum)
3739 {
3740         if (signum == SIGINT || signum == SIGTERM) {
3741                 printf("\nSignal %d received, preparing to exit...\n",
3742                                 signum);
3743 #ifdef RTE_LIBRTE_PDUMP
3744                 /* uninitialize packet capture framework */
3745                 rte_pdump_uninit();
3746 #endif
3747 #ifdef RTE_LIBRTE_LATENCY_STATS
3748                 if (latencystats_enabled != 0)
3749                         rte_latencystats_uninit();
3750 #endif
3751                 force_quit();
3752                 /* Set flag to indicate the force termination. */
3753                 f_quit = 1;
3754                 /* exit with the expected status */
3755                 signal(signum, SIG_DFL);
3756                 kill(getpid(), signum);
3757         }
3758 }
3759
3760 int
3761 main(int argc, char** argv)
3762 {
3763         int diag;
3764         portid_t port_id;
3765         uint16_t count;
3766         int ret;
3767
3768         signal(SIGINT, signal_handler);
3769         signal(SIGTERM, signal_handler);
3770
3771         testpmd_logtype = rte_log_register("testpmd");
3772         if (testpmd_logtype < 0)
3773                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3774         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3775
3776         diag = rte_eal_init(argc, argv);
3777         if (diag < 0)
3778                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3779                          rte_strerror(rte_errno));
3780
3781         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3782                 rte_exit(EXIT_FAILURE,
3783                          "Secondary process type not supported.\n");
3784
3785         ret = register_eth_event_callback();
3786         if (ret != 0)
3787                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3788
3789 #ifdef RTE_LIBRTE_PDUMP
3790         /* initialize packet capture framework */
3791         rte_pdump_init();
3792 #endif
3793
3794         count = 0;
3795         RTE_ETH_FOREACH_DEV(port_id) {
3796                 ports_ids[count] = port_id;
3797                 count++;
3798         }
3799         nb_ports = (portid_t) count;
3800         if (nb_ports == 0)
3801                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3802
3803         /* allocate port structures, and init them */
3804         init_port();
3805
3806         set_def_fwd_config();
3807         if (nb_lcores == 0)
3808                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3809                          "Check the core mask argument\n");
3810
3811         /* Bitrate/latency stats disabled by default */
3812 #ifdef RTE_LIBRTE_BITRATESTATS
3813         bitrate_enabled = 0;
3814 #endif
3815 #ifdef RTE_LIBRTE_LATENCY_STATS
3816         latencystats_enabled = 0;
3817 #endif
3818
3819         /* on FreeBSD, mlockall() is disabled by default */
3820 #ifdef RTE_EXEC_ENV_FREEBSD
3821         do_mlockall = 0;
3822 #else
3823         do_mlockall = 1;
3824 #endif
3825
3826         argc -= diag;
3827         argv += diag;
3828         if (argc > 1)
3829                 launch_args_parse(argc, argv);
3830
3831         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3832                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3833                         strerror(errno));
3834         }
3835
3836         if (tx_first && interactive)
3837                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3838                                 "interactive mode.\n");
3839
3840         if (tx_first && lsc_interrupt) {
3841                 printf("Warning: lsc_interrupt needs to be off when "
3842                                 " using tx_first. Disabling.\n");
3843                 lsc_interrupt = 0;
3844         }
3845
3846         if (!nb_rxq && !nb_txq)
3847                 printf("Warning: Either rx or tx queues should be non-zero\n");
3848
3849         if (nb_rxq > 1 && nb_rxq > nb_txq)
3850                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3851                        "but nb_txq=%d will prevent to fully test it.\n",
3852                        nb_rxq, nb_txq);
3853
3854         init_config();
3855
3856         if (hot_plug) {
3857                 ret = rte_dev_hotplug_handle_enable();
3858                 if (ret) {
3859                         RTE_LOG(ERR, EAL,
3860                                 "fail to enable hotplug handling.");
3861                         return -1;
3862                 }
3863
3864                 ret = rte_dev_event_monitor_start();
3865                 if (ret) {
3866                         RTE_LOG(ERR, EAL,
3867                                 "fail to start device event monitoring.");
3868                         return -1;
3869                 }
3870
3871                 ret = rte_dev_event_callback_register(NULL,
3872                         dev_event_callback, NULL);
3873                 if (ret) {
3874                         RTE_LOG(ERR, EAL,
3875                                 "fail  to register device event callback\n");
3876                         return -1;
3877                 }
3878         }
3879
3880         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3881                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3882
3883         /* set all ports to promiscuous mode by default */
3884         RTE_ETH_FOREACH_DEV(port_id) {
3885                 ret = rte_eth_promiscuous_enable(port_id);
3886                 if (ret != 0)
3887                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3888                                 port_id, rte_strerror(-ret));
3889         }
3890
3891         /* Init metrics library */
3892         rte_metrics_init(rte_socket_id());
3893
3894 #ifdef RTE_LIBRTE_LATENCY_STATS
3895         if (latencystats_enabled != 0) {
3896                 int ret = rte_latencystats_init(1, NULL);
3897                 if (ret)
3898                         printf("Warning: latencystats init()"
3899                                 " returned error %d\n", ret);
3900                 printf("Latencystats running on lcore %d\n",
3901                         latencystats_lcore_id);
3902         }
3903 #endif
3904
3905         /* Setup bitrate stats */
3906 #ifdef RTE_LIBRTE_BITRATESTATS
3907         if (bitrate_enabled != 0) {
3908                 bitrate_data = rte_stats_bitrate_create();
3909                 if (bitrate_data == NULL)
3910                         rte_exit(EXIT_FAILURE,
3911                                 "Could not allocate bitrate data.\n");
3912                 rte_stats_bitrate_reg(bitrate_data);
3913         }
3914 #endif
3915
3916 #ifdef RTE_LIBRTE_CMDLINE
3917         if (strlen(cmdline_filename) != 0)
3918                 cmdline_read_from_file(cmdline_filename);
3919
3920         if (interactive == 1) {
3921                 if (auto_start) {
3922                         printf("Start automatic packet forwarding\n");
3923                         start_packet_forwarding(0);
3924                 }
3925                 prompt();
3926                 pmd_test_exit();
3927         } else
3928 #endif
3929         {
3930                 char c;
3931                 int rc;
3932
3933                 f_quit = 0;
3934
3935                 printf("No commandline core given, start packet forwarding\n");
3936                 start_packet_forwarding(tx_first);
3937                 if (stats_period != 0) {
3938                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3939                         uint64_t timer_period;
3940
3941                         /* Convert to number of cycles */
3942                         timer_period = stats_period * rte_get_timer_hz();
3943
3944                         while (f_quit == 0) {
3945                                 cur_time = rte_get_timer_cycles();
3946                                 diff_time += cur_time - prev_time;
3947
3948                                 if (diff_time >= timer_period) {
3949                                         print_stats();
3950                                         /* Reset the timer */
3951                                         diff_time = 0;
3952                                 }
3953                                 /* Sleep to avoid unnecessary checks */
3954                                 prev_time = cur_time;
3955                                 sleep(1);
3956                         }
3957                 }
3958
3959                 printf("Press enter to exit\n");
3960                 rc = read(0, &c, 1);
3961                 pmd_test_exit();
3962                 if (rc < 0)
3963                         return 1;
3964         }
3965
3966         ret = rte_eal_cleanup();
3967         if (ret != 0)
3968                 rte_exit(EXIT_FAILURE,
3969                          "EAL cleanup failed: %s\n", strerror(-ret));
3970
3971         return EXIT_SUCCESS;
3972 }