app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_NET_IXGBE
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIB_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIB_BITRATESTATS
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIB_LATENCYSTATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use main core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 200         DEFAULT_MBUF_DATA_SIZE
 201 }; /**< Mbuf data space size. */
 202 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 203                                       * specified on command-line. */
 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 205
 206 /*
 207  * In container, it cannot terminate the process which running with 'stats-period'
 208  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 209  */
 210 uint8_t f_quit;
 211
 212 /*
 213  * Configuration of packet segments used to scatter received packets
 214  * if some of split features is configured.
 215  */
 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 217 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 219 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 220
 221 /*
 222  * Configuration of packet segments used by the "txonly" processing engine.
 223  */
 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 226         TXONLY_DEF_PACKET_LEN,
 227 };
 228 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 229
 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 231 /**< Split policy for packets to TX. */
 232
 233 uint8_t txonly_multi_flow;
 234 /**< Whether multiple flows are generated in TXONLY mode. */
 235
 236 uint32_t tx_pkt_times_inter;
 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 238
 239 uint32_t tx_pkt_times_intra;
 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 241
 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 243 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 244
 245 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 246 uint8_t dcb_config = 0;
 247
 248 /* Whether the dcb is in testing status */
 249 uint8_t dcb_test = 0;
 250
 251 /*
 252  * Configurable number of RX/TX queues.
 253  */
 254 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 255 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 256 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 257
 258 /*
 259  * Configurable number of RX/TX ring descriptors.
 260  * Defaults are supplied by drivers via ethdev.
 261  */
 262 #define RTE_TEST_RX_DESC_DEFAULT 0
 263 #define RTE_TEST_TX_DESC_DEFAULT 0
 264 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 265 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 266
 267 #define RTE_PMD_PARAM_UNSET -1
 268 /*
 269  * Configurable values of RX and TX ring threshold registers.
 270  */
 271
 272 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 273 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 274 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 275
 276 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 277 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 278 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 279
 280 /*
 281  * Configurable value of RX free threshold.
 282  */
 283 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 284
 285 /*
 286  * Configurable value of RX drop enable.
 287  */
 288 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 289
 290 /*
 291  * Configurable value of TX free threshold.
 292  */
 293 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 294
 295 /*
 296  * Configurable value of TX RS bit threshold.
 297  */
 298 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 299
 300 /*
 301  * Configurable value of buffered packets before sending.
 302  */
 303 uint16_t noisy_tx_sw_bufsz;
 304
 305 /*
 306  * Configurable value of packet buffer timeout.
 307  */
 308 uint16_t noisy_tx_sw_buf_flush_time;
 309
 310 /*
 311  * Configurable value for size of VNF internal memory area
 312  * used for simulating noisy neighbour behaviour
 313  */
 314 uint64_t noisy_lkup_mem_sz;
 315
 316 /*
 317  * Configurable value of number of random writes done in
 318  * VNF simulation memory area.
 319  */
 320 uint64_t noisy_lkup_num_writes;
 321
 322 /*
 323  * Configurable value of number of random reads done in
 324  * VNF simulation memory area.
 325  */
 326 uint64_t noisy_lkup_num_reads;
 327
 328 /*
 329  * Configurable value of number of random reads/writes done in
 330  * VNF simulation memory area.
 331  */
 332 uint64_t noisy_lkup_num_reads_writes;
 333
 334 /*
 335  * Receive Side Scaling (RSS) configuration.
 336  */
 337 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 338
 339 /*
 340  * Port topology configuration
 341  */
 342 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 343
 344 /*
 345  * Avoids to flush all the RX streams before starts forwarding.
 346  */
 347 uint8_t no_flush_rx = 0; /* flush by default */
 348
 349 /*
 350  * Flow API isolated mode.
 351  */
 352 uint8_t flow_isolate_all;
 353
 354 /*
 355  * Avoids to check link status when starting/stopping a port.
 356  */
 357 uint8_t no_link_check = 0; /* check by default */
 358
 359 /*
 360  * Don't automatically start all ports in interactive mode.
 361  */
 362 uint8_t no_device_start = 0;
 363
 364 /*
 365  * Enable link status change notification
 366  */
 367 uint8_t lsc_interrupt = 1; /* enabled by default */
 368
 369 /*
 370  * Enable device removal notification.
 371  */
 372 uint8_t rmv_interrupt = 1; /* enabled by default */
 373
 374 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 375
 376 /* After attach, port setup is called on event or by iterator */
 377 bool setup_on_probe_event = true;
 378
 379 /* Clear ptypes on port initialization. */
 380 uint8_t clear_ptypes = true;
 381
 382 /* Hairpin ports configuration mode. */
 383 uint16_t hairpin_mode;
 384
 385 /* Pretty printing of ethdev events */
 386 static const char * const eth_event_desc[] = {
 387         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 388         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 389         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 390         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 391         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 392         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 393         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 394         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 395         [RTE_ETH_EVENT_NEW] = "device probed",
 396         [RTE_ETH_EVENT_DESTROY] = "device released",
 397         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 398         [RTE_ETH_EVENT_MAX] = NULL,
 399 };
 400
 401 /*
 402  * Display or mask ether events
 403  * Default to all events except VF_MBOX
 404  */
 405 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 406                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 407                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 408                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 413 /*
 414  * Decide if all memory are locked for performance.
 415  */
 416 int do_mlockall = 0;
 417
 418 /*
 419  * NIC bypass mode configuration options.
 420  */
 421
 422 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 423 /* The NIC bypass watchdog timeout. */
 424 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 425 #endif
 426
 427
 428 #ifdef RTE_LIB_LATENCYSTATS
 429
 430 /*
 431  * Set when latency stats is enabled in the commandline
 432  */
 433 uint8_t latencystats_enabled;
 434
 435 /*
 436  * Lcore ID to serive latency statistics.
 437  */
 438 lcoreid_t latencystats_lcore_id = -1;
 439
 440 #endif
 441
 442 /*
 443  * Ethernet device configuration.
 444  */
 445 struct rte_eth_rxmode rx_mode = {
 446         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 447                 /**< Default maximum frame length. */
 448 };
 449
 450 struct rte_eth_txmode tx_mode = {
 451         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 452 };
 453
 454 struct rte_fdir_conf fdir_conf = {
 455         .mode = RTE_FDIR_MODE_NONE,
 456         .pballoc = RTE_FDIR_PBALLOC_64K,
 457         .status = RTE_FDIR_REPORT_STATUS,
 458         .mask = {
 459                 .vlan_tci_mask = 0xFFEF,
 460                 .ipv4_mask     = {
 461                         .src_ip = 0xFFFFFFFF,
 462                         .dst_ip = 0xFFFFFFFF,
 463                 },
 464                 .ipv6_mask     = {
 465                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 466                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 467                 },
 468                 .src_port_mask = 0xFFFF,
 469                 .dst_port_mask = 0xFFFF,
 470                 .mac_addr_byte_mask = 0xFF,
 471                 .tunnel_type_mask = 1,
 472                 .tunnel_id_mask = 0xFFFFFFFF,
 473         },
 474         .drop_queue = 127,
 475 };
 476
 477 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 478
 479 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 480 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 481
 482 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 483 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 484
 485 uint16_t nb_tx_queue_stats_mappings = 0;
 486 uint16_t nb_rx_queue_stats_mappings = 0;
 487
 488 /*
 489  * Display zero values by default for xstats
 490  */
 491 uint8_t xstats_hide_zero;
 492
 493 /*
 494  * Measure of CPU cycles disabled by default
 495  */
 496 uint8_t record_core_cycles;
 497
 498 /*
 499  * Display of RX and TX bursts disabled by default
 500  */
 501 uint8_t record_burst_stats;
 502
 503 unsigned int num_sockets = 0;
 504 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 505
 506 #ifdef RTE_LIB_BITRATESTATS
 507 /* Bitrate statistics */
 508 struct rte_stats_bitrates *bitrate_data;
 509 lcoreid_t bitrate_lcore_id;
 510 uint8_t bitrate_enabled;
 511 #endif
 512
 513 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 514 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 515
 516 /*
 517  * hexadecimal bitmask of RX mq mode can be enabled.
 518  */
 519 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 520
 521 /* Forward function declarations */
 522 static void setup_attached_port(portid_t pi);
 523 static void map_port_queue_stats_mapping_registers(portid_t pi,
 524                                                    struct rte_port *port);
 525 static void check_all_ports_link_status(uint32_t port_mask);
 526 static int eth_event_callback(portid_t port_id,
 527                               enum rte_eth_event_type type,
 528                               void *param, void *ret_param);
 529 static void dev_event_callback(const char *device_name,
 530                                 enum rte_dev_event_type type,
 531                                 void *param);
 532
 533 /*
 534  * Check if all the ports are started.
 535  * If yes, return positive value. If not, return zero.
 536  */
 537 static int all_ports_started(void);
 538
 539 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 540 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 541
 542 /* Holds the registered mbuf dynamic flags names. */
 543 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 544
 545 /*
 546  * Helper function to check if socket is already discovered.
 547  * If yes, return positive value. If not, return zero.
 548  */
 549 int
 550 new_socket_id(unsigned int socket_id)
 551 {
 552         unsigned int i;
 553
 554         for (i = 0; i < num_sockets; i++) {
 555                 if (socket_ids[i] == socket_id)
 556                         return 0;
 557         }
 558         return 1;
 559 }
 560
 561 /*
 562  * Setup default configuration.
 563  */
 564 static void
 565 set_default_fwd_lcores_config(void)
 566 {
 567         unsigned int i;
 568         unsigned int nb_lc;
 569         unsigned int sock_num;
 570
 571         nb_lc = 0;
 572         for (i = 0; i < RTE_MAX_LCORE; i++) {
 573                 if (!rte_lcore_is_enabled(i))
 574                         continue;
 575                 sock_num = rte_lcore_to_socket_id(i);
 576                 if (new_socket_id(sock_num)) {
 577                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 578                                 rte_exit(EXIT_FAILURE,
 579                                          "Total sockets greater than %u\n",
 580                                          RTE_MAX_NUMA_NODES);
 581                         }
 582                         socket_ids[num_sockets++] = sock_num;
 583                 }
 584                 if (i == rte_get_main_lcore())
 585                         continue;
 586                 fwd_lcores_cpuids[nb_lc++] = i;
 587         }
 588         nb_lcores = (lcoreid_t) nb_lc;
 589         nb_cfg_lcores = nb_lcores;
 590         nb_fwd_lcores = 1;
 591 }
 592
 593 static void
 594 set_def_peer_eth_addrs(void)
 595 {
 596         portid_t i;
 597
 598         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 599                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 600                 peer_eth_addrs[i].addr_bytes[5] = i;
 601         }
 602 }
 603
 604 static void
 605 set_default_fwd_ports_config(void)
 606 {
 607         portid_t pt_id;
 608         int i = 0;
 609
 610         RTE_ETH_FOREACH_DEV(pt_id) {
 611                 fwd_ports_ids[i++] = pt_id;
 612
 613                 /* Update sockets info according to the attached device */
 614                 int socket_id = rte_eth_dev_socket_id(pt_id);
 615                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 616                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 617                                 rte_exit(EXIT_FAILURE,
 618                                          "Total sockets greater than %u\n",
 619                                          RTE_MAX_NUMA_NODES);
 620                         }
 621                         socket_ids[num_sockets++] = socket_id;
 622                 }
 623         }
 624
 625         nb_cfg_ports = nb_ports;
 626         nb_fwd_ports = nb_ports;
 627 }
 628
 629 void
 630 set_def_fwd_config(void)
 631 {
 632         set_default_fwd_lcores_config();
 633         set_def_peer_eth_addrs();
 634         set_default_fwd_ports_config();
 635 }
 636
 637 /* extremely pessimistic estimation of memory required to create a mempool */
 638 static int
 639 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 640 {
 641         unsigned int n_pages, mbuf_per_pg, leftover;
 642         uint64_t total_mem, mbuf_mem, obj_sz;
 643
 644         /* there is no good way to predict how much space the mempool will
 645          * occupy because it will allocate chunks on the fly, and some of those
 646          * will come from default DPDK memory while some will come from our
 647          * external memory, so just assume 128MB will be enough for everyone.
 648          */
 649         uint64_t hdr_mem = 128 << 20;
 650
 651         /* account for possible non-contiguousness */
 652         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 653         if (obj_sz > pgsz) {
 654                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 655                 return -1;
 656         }
 657
 658         mbuf_per_pg = pgsz / obj_sz;
 659         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 660         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 661
 662         mbuf_mem = n_pages * pgsz;
 663
 664         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 665
 666         if (total_mem > SIZE_MAX) {
 667                 TESTPMD_LOG(ERR, "Memory size too big\n");
 668                 return -1;
 669         }
 670         *out = (size_t)total_mem;
 671
 672         return 0;
 673 }
 674
 675 static int
 676 pagesz_flags(uint64_t page_sz)
 677 {
 678         /* as per mmap() manpage, all page sizes are log2 of page size
 679          * shifted by MAP_HUGE_SHIFT
 680          */
 681         int log2 = rte_log2_u64(page_sz);
 682
 683         return (log2 << HUGE_SHIFT);
 684 }
 685
 686 static void *
 687 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 688 {
 689         void *addr;
 690         int flags;
 691
 692         /* allocate anonymous hugepages */
 693         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 694         if (huge)
 695                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 696
 697         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 698         if (addr == MAP_FAILED)
 699                 return NULL;
 700
 701         return addr;
 702 }
 703
 704 struct extmem_param {
 705         void *addr;
 706         size_t len;
 707         size_t pgsz;
 708         rte_iova_t *iova_table;
 709         unsigned int iova_table_len;
 710 };
 711
 712 static int
 713 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 714                 bool huge)
 715 {
 716         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 717                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 718         unsigned int cur_page, n_pages, pgsz_idx;
 719         size_t mem_sz, cur_pgsz;
 720         rte_iova_t *iovas = NULL;
 721         void *addr;
 722         int ret;
 723
 724         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 725                 /* skip anything that is too big */
 726                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 727                         continue;
 728
 729                 cur_pgsz = pgsizes[pgsz_idx];
 730
 731                 /* if we were told not to allocate hugepages, override */
 732                 if (!huge)
 733                         cur_pgsz = sysconf(_SC_PAGESIZE);
 734
 735                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 736                 if (ret < 0) {
 737                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 738                         return -1;
 739                 }
 740
 741                 /* allocate our memory */
 742                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 743
 744                 /* if we couldn't allocate memory with a specified page size,
 745                  * that doesn't mean we can't do it with other page sizes, so
 746                  * try another one.
 747                  */
 748                 if (addr == NULL)
 749                         continue;
 750
 751                 /* store IOVA addresses for every page in this memory area */
 752                 n_pages = mem_sz / cur_pgsz;
 753
 754                 iovas = malloc(sizeof(*iovas) * n_pages);
 755
 756                 if (iovas == NULL) {
 757                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 758                         goto fail;
 759                 }
 760                 /* lock memory if it's not huge pages */
 761                 if (!huge)
 762                         mlock(addr, mem_sz);
 763
 764                 /* populate IOVA addresses */
 765                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 766                         rte_iova_t iova;
 767                         size_t offset;
 768                         void *cur;
 769
 770                         offset = cur_pgsz * cur_page;
 771                         cur = RTE_PTR_ADD(addr, offset);
 772
 773                         /* touch the page before getting its IOVA */
 774                         *(volatile char *)cur = 0;
 775
 776                         iova = rte_mem_virt2iova(cur);
 777
 778                         iovas[cur_page] = iova;
 779                 }
 780
 781                 break;
 782         }
 783         /* if we couldn't allocate anything */
 784         if (iovas == NULL)
 785                 return -1;
 786
 787         param->addr = addr;
 788         param->len = mem_sz;
 789         param->pgsz = cur_pgsz;
 790         param->iova_table = iovas;
 791         param->iova_table_len = n_pages;
 792
 793         return 0;
 794 fail:
 795         if (iovas)
 796                 free(iovas);
 797         if (addr)
 798                 munmap(addr, mem_sz);
 799
 800         return -1;
 801 }
 802
 803 static int
 804 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 805 {
 806         struct extmem_param param;
 807         int socket_id, ret;
 808
 809         memset(&param, 0, sizeof(param));
 810
 811         /* check if our heap exists */
 812         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 813         if (socket_id < 0) {
 814                 /* create our heap */
 815                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 816                 if (ret < 0) {
 817                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 818                         return -1;
 819                 }
 820         }
 821
 822         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 823         if (ret < 0) {
 824                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 825                 return -1;
 826         }
 827
 828         /* we now have a valid memory area, so add it to heap */
 829         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 830                         param.addr, param.len, param.iova_table,
 831                         param.iova_table_len, param.pgsz);
 832
 833         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 834
 835         /* not needed any more */
 836         free(param.iova_table);
 837
 838         if (ret < 0) {
 839                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 840                 munmap(param.addr, param.len);
 841                 return -1;
 842         }
 843
 844         /* success */
 845
 846         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 847                         param.len >> 20);
 848
 849         return 0;
 850 }
 851 static void
 852 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 853              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 854 {
 855         uint16_t pid = 0;
 856         int ret;
 857
 858         RTE_ETH_FOREACH_DEV(pid) {
 859                 struct rte_eth_dev *dev =
 860                         &rte_eth_devices[pid];
 861
 862                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 863                                         memhdr->len);
 864                 if (ret) {
 865                         TESTPMD_LOG(DEBUG,
 866                                     "unable to DMA unmap addr 0x%p "
 867                                     "for device %s\n",
 868                                     memhdr->addr, dev->data->name);
 869                 }
 870         }
 871         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 872         if (ret) {
 873                 TESTPMD_LOG(DEBUG,
 874                             "unable to un-register addr 0x%p\n", memhdr->addr);
 875         }
 876 }
 877
 878 static void
 879 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 880            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 881 {
 882         uint16_t pid = 0;
 883         size_t page_size = sysconf(_SC_PAGESIZE);
 884         int ret;
 885
 886         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 887                                   page_size);
 888         if (ret) {
 889                 TESTPMD_LOG(DEBUG,
 890                             "unable to register addr 0x%p\n", memhdr->addr);
 891                 return;
 892         }
 893         RTE_ETH_FOREACH_DEV(pid) {
 894                 struct rte_eth_dev *dev =
 895                         &rte_eth_devices[pid];
 896
 897                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 898                                       memhdr->len);
 899                 if (ret) {
 900                         TESTPMD_LOG(DEBUG,
 901                                     "unable to DMA map addr 0x%p "
 902                                     "for device %s\n",
 903                                     memhdr->addr, dev->data->name);
 904                 }
 905         }
 906 }
 907
 908 static unsigned int
 909 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 910             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 911 {
 912         struct rte_pktmbuf_extmem *xmem;
 913         unsigned int ext_num, zone_num, elt_num;
 914         uint16_t elt_size;
 915
 916         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 917         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 918         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 919
 920         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 921         if (xmem == NULL) {
 922                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 923                                  "external buffer descriptors\n");
 924                 *ext_mem = NULL;
 925                 return 0;
 926         }
 927         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 928                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 929                 const struct rte_memzone *mz;
 930                 char mz_name[RTE_MEMZONE_NAMESIZE];
 931                 int ret;
 932
 933                 ret = snprintf(mz_name, sizeof(mz_name),
 934                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 935                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 936                         errno = ENAMETOOLONG;
 937                         ext_num = 0;
 938                         break;
 939                 }
 940                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 941                                                  socket_id,
 942                                                  RTE_MEMZONE_IOVA_CONTIG |
 943                                                  RTE_MEMZONE_1GB |
 944                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 945                                                  EXTBUF_ZONE_SIZE);
 946                 if (mz == NULL) {
 947                         /*
 948                          * The caller exits on external buffer creation
 949                          * error, so there is no need to free memzones.
 950                          */
 951                         errno = ENOMEM;
 952                         ext_num = 0;
 953                         break;
 954                 }
 955                 xseg->buf_ptr = mz->addr;
 956                 xseg->buf_iova = mz->iova;
 957                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 958                 xseg->elt_size = elt_size;
 959         }
 960         if (ext_num == 0 && xmem != NULL) {
 961                 free(xmem);
 962                 xmem = NULL;
 963         }
 964         *ext_mem = xmem;
 965         return ext_num;
 966 }
 967
 968 /*
 969  * Configuration initialisation done once at init time.
 970  */
 971 static struct rte_mempool *
 972 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 973                  unsigned int socket_id, uint16_t size_idx)
 974 {
 975         char pool_name[RTE_MEMPOOL_NAMESIZE];
 976         struct rte_mempool *rte_mp = NULL;
 977         uint32_t mb_size;
 978
 979         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 980         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 981
 982         TESTPMD_LOG(INFO,
 983                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 984                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 985
 986         switch (mp_alloc_type) {
 987         case MP_ALLOC_NATIVE:
 988                 {
 989                         /* wrapper to rte_mempool_create() */
 990                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 991                                         rte_mbuf_best_mempool_ops());
 992                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 993                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 994                         break;
 995                 }
 996         case MP_ALLOC_ANON:
 997                 {
 998                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 999                                 mb_size, (unsigned int) mb_mempool_cache,
1000                                 sizeof(struct rte_pktmbuf_pool_private),
1001                                 socket_id, mempool_flags);
1002                         if (rte_mp == NULL)
1003                                 goto err;
1004
1005                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1006                                 rte_mempool_free(rte_mp);
1007                                 rte_mp = NULL;
1008                                 goto err;
1009                         }
1010                         rte_pktmbuf_pool_init(rte_mp, NULL);
1011                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1012                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1013                         break;
1014                 }
1015         case MP_ALLOC_XMEM:
1016         case MP_ALLOC_XMEM_HUGE:
1017                 {
1018                         int heap_socket;
1019                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1020
1021                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1022                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1023
1024                         heap_socket =
1025                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1026                         if (heap_socket < 0)
1027                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1028
1029                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1030                                         rte_mbuf_best_mempool_ops());
1031                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1032                                         mb_mempool_cache, 0, mbuf_seg_size,
1033                                         heap_socket);
1034                         break;
1035                 }
1036         case MP_ALLOC_XBUF:
1037                 {
1038                         struct rte_pktmbuf_extmem *ext_mem;
1039                         unsigned int ext_num;
1040
1041                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1042                                                socket_id, pool_name, &ext_mem);
1043                         if (ext_num == 0)
1044                                 rte_exit(EXIT_FAILURE,
1045                                          "Can't create pinned data buffers\n");
1046
1047                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1048                                         rte_mbuf_best_mempool_ops());
1049                         rte_mp = rte_pktmbuf_pool_create_extbuf
1050                                         (pool_name, nb_mbuf, mb_mempool_cache,
1051                                          0, mbuf_seg_size, socket_id,
1052                                          ext_mem, ext_num);
1053                         free(ext_mem);
1054                         break;
1055                 }
1056         default:
1057                 {
1058                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1059                 }
1060         }
1061
1062 err:
1063         if (rte_mp == NULL) {
1064                 rte_exit(EXIT_FAILURE,
1065                         "Creation of mbuf pool for socket %u failed: %s\n",
1066                         socket_id, rte_strerror(rte_errno));
1067         } else if (verbose_level > 0) {
1068                 rte_mempool_dump(stdout, rte_mp);
1069         }
1070         return rte_mp;
1071 }
1072
1073 /*
1074  * Check given socket id is valid or not with NUMA mode,
1075  * if valid, return 0, else return -1
1076  */
1077 static int
1078 check_socket_id(const unsigned int socket_id)
1079 {
1080         static int warning_once = 0;
1081
1082         if (new_socket_id(socket_id)) {
1083                 if (!warning_once && numa_support)
1084                         printf("Warning: NUMA should be configured manually by"
1085                                " using --port-numa-config and"
1086                                " --ring-numa-config parameters along with"
1087                                " --numa.\n");
1088                 warning_once = 1;
1089                 return -1;
1090         }
1091         return 0;
1092 }
1093
1094 /*
1095  * Get the allowed maximum number of RX queues.
1096  * *pid return the port id which has minimal value of
1097  * max_rx_queues in all ports.
1098  */
1099 queueid_t
1100 get_allowed_max_nb_rxq(portid_t *pid)
1101 {
1102         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1103         bool max_rxq_valid = false;
1104         portid_t pi;
1105         struct rte_eth_dev_info dev_info;
1106
1107         RTE_ETH_FOREACH_DEV(pi) {
1108                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1109                         continue;
1110
1111                 max_rxq_valid = true;
1112                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1113                         allowed_max_rxq = dev_info.max_rx_queues;
1114                         *pid = pi;
1115                 }
1116         }
1117         return max_rxq_valid ? allowed_max_rxq : 0;
1118 }
1119
1120 /*
1121  * Check input rxq is valid or not.
1122  * If input rxq is not greater than any of maximum number
1123  * of RX queues of all ports, it is valid.
1124  * if valid, return 0, else return -1
1125  */
1126 int
1127 check_nb_rxq(queueid_t rxq)
1128 {
1129         queueid_t allowed_max_rxq;
1130         portid_t pid = 0;
1131
1132         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1133         if (rxq > allowed_max_rxq) {
1134                 printf("Fail: input rxq (%u) can't be greater "
1135                        "than max_rx_queues (%u) of port %u\n",
1136                        rxq,
1137                        allowed_max_rxq,
1138                        pid);
1139                 return -1;
1140         }
1141         return 0;
1142 }
1143
1144 /*
1145  * Get the allowed maximum number of TX queues.
1146  * *pid return the port id which has minimal value of
1147  * max_tx_queues in all ports.
1148  */
1149 queueid_t
1150 get_allowed_max_nb_txq(portid_t *pid)
1151 {
1152         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1153         bool max_txq_valid = false;
1154         portid_t pi;
1155         struct rte_eth_dev_info dev_info;
1156
1157         RTE_ETH_FOREACH_DEV(pi) {
1158                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1159                         continue;
1160
1161                 max_txq_valid = true;
1162                 if (dev_info.max_tx_queues < allowed_max_txq) {
1163                         allowed_max_txq = dev_info.max_tx_queues;
1164                         *pid = pi;
1165                 }
1166         }
1167         return max_txq_valid ? allowed_max_txq : 0;
1168 }
1169
1170 /*
1171  * Check input txq is valid or not.
1172  * If input txq is not greater than any of maximum number
1173  * of TX queues of all ports, it is valid.
1174  * if valid, return 0, else return -1
1175  */
1176 int
1177 check_nb_txq(queueid_t txq)
1178 {
1179         queueid_t allowed_max_txq;
1180         portid_t pid = 0;
1181
1182         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1183         if (txq > allowed_max_txq) {
1184                 printf("Fail: input txq (%u) can't be greater "
1185                        "than max_tx_queues (%u) of port %u\n",
1186                        txq,
1187                        allowed_max_txq,
1188                        pid);
1189                 return -1;
1190         }
1191         return 0;
1192 }
1193
1194 /*
1195  * Get the allowed maximum number of RXDs of every rx queue.
1196  * *pid return the port id which has minimal value of
1197  * max_rxd in all queues of all ports.
1198  */
1199 static uint16_t
1200 get_allowed_max_nb_rxd(portid_t *pid)
1201 {
1202         uint16_t allowed_max_rxd = UINT16_MAX;
1203         portid_t pi;
1204         struct rte_eth_dev_info dev_info;
1205
1206         RTE_ETH_FOREACH_DEV(pi) {
1207                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1208                         continue;
1209
1210                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1211                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1212                         *pid = pi;
1213                 }
1214         }
1215         return allowed_max_rxd;
1216 }
1217
1218 /*
1219  * Get the allowed minimal number of RXDs of every rx queue.
1220  * *pid return the port id which has minimal value of
1221  * min_rxd in all queues of all ports.
1222  */
1223 static uint16_t
1224 get_allowed_min_nb_rxd(portid_t *pid)
1225 {
1226         uint16_t allowed_min_rxd = 0;
1227         portid_t pi;
1228         struct rte_eth_dev_info dev_info;
1229
1230         RTE_ETH_FOREACH_DEV(pi) {
1231                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1232                         continue;
1233
1234                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1235                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1236                         *pid = pi;
1237                 }
1238         }
1239
1240         return allowed_min_rxd;
1241 }
1242
1243 /*
1244  * Check input rxd is valid or not.
1245  * If input rxd is not greater than any of maximum number
1246  * of RXDs of every Rx queues and is not less than any of
1247  * minimal number of RXDs of every Rx queues, it is valid.
1248  * if valid, return 0, else return -1
1249  */
1250 int
1251 check_nb_rxd(queueid_t rxd)
1252 {
1253         uint16_t allowed_max_rxd;
1254         uint16_t allowed_min_rxd;
1255         portid_t pid = 0;
1256
1257         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1258         if (rxd > allowed_max_rxd) {
1259                 printf("Fail: input rxd (%u) can't be greater "
1260                        "than max_rxds (%u) of port %u\n",
1261                        rxd,
1262                        allowed_max_rxd,
1263                        pid);
1264                 return -1;
1265         }
1266
1267         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1268         if (rxd < allowed_min_rxd) {
1269                 printf("Fail: input rxd (%u) can't be less "
1270                        "than min_rxds (%u) of port %u\n",
1271                        rxd,
1272                        allowed_min_rxd,
1273                        pid);
1274                 return -1;
1275         }
1276
1277         return 0;
1278 }
1279
1280 /*
1281  * Get the allowed maximum number of TXDs of every rx queues.
1282  * *pid return the port id which has minimal value of
1283  * max_txd in every tx queue.
1284  */
1285 static uint16_t
1286 get_allowed_max_nb_txd(portid_t *pid)
1287 {
1288         uint16_t allowed_max_txd = UINT16_MAX;
1289         portid_t pi;
1290         struct rte_eth_dev_info dev_info;
1291
1292         RTE_ETH_FOREACH_DEV(pi) {
1293                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1294                         continue;
1295
1296                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1297                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1298                         *pid = pi;
1299                 }
1300         }
1301         return allowed_max_txd;
1302 }
1303
1304 /*
1305  * Get the allowed maximum number of TXDs of every tx queues.
1306  * *pid return the port id which has minimal value of
1307  * min_txd in every tx queue.
1308  */
1309 static uint16_t
1310 get_allowed_min_nb_txd(portid_t *pid)
1311 {
1312         uint16_t allowed_min_txd = 0;
1313         portid_t pi;
1314         struct rte_eth_dev_info dev_info;
1315
1316         RTE_ETH_FOREACH_DEV(pi) {
1317                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1318                         continue;
1319
1320                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1321                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1322                         *pid = pi;
1323                 }
1324         }
1325
1326         return allowed_min_txd;
1327 }
1328
1329 /*
1330  * Check input txd is valid or not.
1331  * If input txd is not greater than any of maximum number
1332  * of TXDs of every Rx queues, it is valid.
1333  * if valid, return 0, else return -1
1334  */
1335 int
1336 check_nb_txd(queueid_t txd)
1337 {
1338         uint16_t allowed_max_txd;
1339         uint16_t allowed_min_txd;
1340         portid_t pid = 0;
1341
1342         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1343         if (txd > allowed_max_txd) {
1344                 printf("Fail: input txd (%u) can't be greater "
1345                        "than max_txds (%u) of port %u\n",
1346                        txd,
1347                        allowed_max_txd,
1348                        pid);
1349                 return -1;
1350         }
1351
1352         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1353         if (txd < allowed_min_txd) {
1354                 printf("Fail: input txd (%u) can't be less "
1355                        "than min_txds (%u) of port %u\n",
1356                        txd,
1357                        allowed_min_txd,
1358                        pid);
1359                 return -1;
1360         }
1361         return 0;
1362 }
1363
1364
1365 /*
1366  * Get the allowed maximum number of hairpin queues.
1367  * *pid return the port id which has minimal value of
1368  * max_hairpin_queues in all ports.
1369  */
1370 queueid_t
1371 get_allowed_max_nb_hairpinq(portid_t *pid)
1372 {
1373         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1374         portid_t pi;
1375         struct rte_eth_hairpin_cap cap;
1376
1377         RTE_ETH_FOREACH_DEV(pi) {
1378                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1379                         *pid = pi;
1380                         return 0;
1381                 }
1382                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1383                         allowed_max_hairpinq = cap.max_nb_queues;
1384                         *pid = pi;
1385                 }
1386         }
1387         return allowed_max_hairpinq;
1388 }
1389
1390 /*
1391  * Check input hairpin is valid or not.
1392  * If input hairpin is not greater than any of maximum number
1393  * of hairpin queues of all ports, it is valid.
1394  * if valid, return 0, else return -1
1395  */
1396 int
1397 check_nb_hairpinq(queueid_t hairpinq)
1398 {
1399         queueid_t allowed_max_hairpinq;
1400         portid_t pid = 0;
1401
1402         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1403         if (hairpinq > allowed_max_hairpinq) {
1404                 printf("Fail: input hairpin (%u) can't be greater "
1405                        "than max_hairpin_queues (%u) of port %u\n",
1406                        hairpinq, allowed_max_hairpinq, pid);
1407                 return -1;
1408         }
1409         return 0;
1410 }
1411
1412 static void
1413 init_config(void)
1414 {
1415         portid_t pid;
1416         struct rte_port *port;
1417         struct rte_mempool *mbp;
1418         unsigned int nb_mbuf_per_pool;
1419         lcoreid_t  lc_id;
1420         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1421         struct rte_gro_param gro_param;
1422         uint32_t gso_types;
1423         uint16_t data_size;
1424         bool warning = 0;
1425         int k;
1426         int ret;
1427
1428         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1429
1430         /* Configuration of logical cores. */
1431         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1432                                 sizeof(struct fwd_lcore *) * nb_lcores,
1433                                 RTE_CACHE_LINE_SIZE);
1434         if (fwd_lcores == NULL) {
1435                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1436                                                         "failed\n", nb_lcores);
1437         }
1438         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1439                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1440                                                sizeof(struct fwd_lcore),
1441                                                RTE_CACHE_LINE_SIZE);
1442                 if (fwd_lcores[lc_id] == NULL) {
1443                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1444                                                                 "failed\n");
1445                 }
1446                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1447         }
1448
1449         RTE_ETH_FOREACH_DEV(pid) {
1450                 port = &ports[pid];
1451                 /* Apply default TxRx configuration for all ports */
1452                 port->dev_conf.txmode = tx_mode;
1453                 port->dev_conf.rxmode = rx_mode;
1454
1455                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1456                 if (ret != 0)
1457                         rte_exit(EXIT_FAILURE,
1458                                  "rte_eth_dev_info_get() failed\n");
1459
1460                 if (!(port->dev_info.tx_offload_capa &
1461                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1462                         port->dev_conf.txmode.offloads &=
1463                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1464                 if (numa_support) {
1465                         if (port_numa[pid] != NUMA_NO_CONFIG)
1466                                 port_per_socket[port_numa[pid]]++;
1467                         else {
1468                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1469
1470                                 /*
1471                                  * if socket_id is invalid,
1472                                  * set to the first available socket.
1473                                  */
1474                                 if (check_socket_id(socket_id) < 0)
1475                                         socket_id = socket_ids[0];
1476                                 port_per_socket[socket_id]++;
1477                         }
1478                 }
1479
1480                 /* Apply Rx offloads configuration */
1481                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1482                         port->rx_conf[k].offloads =
1483                                 port->dev_conf.rxmode.offloads;
1484                 /* Apply Tx offloads configuration */
1485                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1486                         port->tx_conf[k].offloads =
1487                                 port->dev_conf.txmode.offloads;
1488
1489                 /* set flag to initialize port/queue */
1490                 port->need_reconfig = 1;
1491                 port->need_reconfig_queues = 1;
1492                 port->tx_metadata = 0;
1493
1494                 /* Check for maximum number of segments per MTU. Accordingly
1495                  * update the mbuf data size.
1496                  */
1497                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1498                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1499                         data_size = rx_mode.max_rx_pkt_len /
1500                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1501
1502                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1503                                                         mbuf_data_size[0]) {
1504                                 mbuf_data_size[0] = data_size +
1505                                                  RTE_PKTMBUF_HEADROOM;
1506                                 warning = 1;
1507                         }
1508                 }
1509         }
1510
1511         if (warning)
1512                 TESTPMD_LOG(WARNING,
1513                             "Configured mbuf size of the first segment %hu\n",
1514                             mbuf_data_size[0]);
1515         /*
1516          * Create pools of mbuf.
1517          * If NUMA support is disabled, create a single pool of mbuf in
1518          * socket 0 memory by default.
1519          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1520          *
1521          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1522          * nb_txd can be configured at run time.
1523          */
1524         if (param_total_num_mbufs)
1525                 nb_mbuf_per_pool = param_total_num_mbufs;
1526         else {
1527                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1528                         (nb_lcores * mb_mempool_cache) +
1529                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1530                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1531         }
1532
1533         if (numa_support) {
1534                 uint8_t i, j;
1535
1536                 for (i = 0; i < num_sockets; i++)
1537                         for (j = 0; j < mbuf_data_size_n; j++)
1538                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1539                                         mbuf_pool_create(mbuf_data_size[j],
1540                                                           nb_mbuf_per_pool,
1541                                                           socket_ids[i], j);
1542         } else {
1543                 uint8_t i;
1544
1545                 for (i = 0; i < mbuf_data_size_n; i++)
1546                         mempools[i] = mbuf_pool_create
1547                                         (mbuf_data_size[i],
1548                                          nb_mbuf_per_pool,
1549                                          socket_num == UMA_NO_CONFIG ?
1550                                          0 : socket_num, i);
1551         }
1552
1553         init_port_config();
1554
1555         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1556                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1557         /*
1558          * Records which Mbuf pool to use by each logical core, if needed.
1559          */
1560         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1561                 mbp = mbuf_pool_find(
1562                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1563
1564                 if (mbp == NULL)
1565                         mbp = mbuf_pool_find(0, 0);
1566                 fwd_lcores[lc_id]->mbp = mbp;
1567                 /* initialize GSO context */
1568                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1569                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1570                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1571                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1572                         RTE_ETHER_CRC_LEN;
1573                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1574         }
1575
1576         /* Configuration of packet forwarding streams. */
1577         if (init_fwd_streams() < 0)
1578                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1579
1580         fwd_config_setup();
1581
1582         /* create a gro context for each lcore */
1583         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1584         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1585         gro_param.max_item_per_flow = MAX_PKT_BURST;
1586         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1587                 gro_param.socket_id = rte_lcore_to_socket_id(
1588                                 fwd_lcores_cpuids[lc_id]);
1589                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1590                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1591                         rte_exit(EXIT_FAILURE,
1592                                         "rte_gro_ctx_create() failed\n");
1593                 }
1594         }
1595 }
1596
1597
1598 void
1599 reconfig(portid_t new_port_id, unsigned socket_id)
1600 {
1601         struct rte_port *port;
1602         int ret;
1603
1604         /* Reconfiguration of Ethernet ports. */
1605         port = &ports[new_port_id];
1606
1607         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1608         if (ret != 0)
1609                 return;
1610
1611         /* set flag to initialize port/queue */
1612         port->need_reconfig = 1;
1613         port->need_reconfig_queues = 1;
1614         port->socket_id = socket_id;
1615
1616         init_port_config();
1617 }
1618
1619
1620 int
1621 init_fwd_streams(void)
1622 {
1623         portid_t pid;
1624         struct rte_port *port;
1625         streamid_t sm_id, nb_fwd_streams_new;
1626         queueid_t q;
1627
1628         /* set socket id according to numa or not */
1629         RTE_ETH_FOREACH_DEV(pid) {
1630                 port = &ports[pid];
1631                 if (nb_rxq > port->dev_info.max_rx_queues) {
1632                         printf("Fail: nb_rxq(%d) is greater than "
1633                                 "max_rx_queues(%d)\n", nb_rxq,
1634                                 port->dev_info.max_rx_queues);
1635                         return -1;
1636                 }
1637                 if (nb_txq > port->dev_info.max_tx_queues) {
1638                         printf("Fail: nb_txq(%d) is greater than "
1639                                 "max_tx_queues(%d)\n", nb_txq,
1640                                 port->dev_info.max_tx_queues);
1641                         return -1;
1642                 }
1643                 if (numa_support) {
1644                         if (port_numa[pid] != NUMA_NO_CONFIG)
1645                                 port->socket_id = port_numa[pid];
1646                         else {
1647                                 port->socket_id = rte_eth_dev_socket_id(pid);
1648
1649                                 /*
1650                                  * if socket_id is invalid,
1651                                  * set to the first available socket.
1652                                  */
1653                                 if (check_socket_id(port->socket_id) < 0)
1654                                         port->socket_id = socket_ids[0];
1655                         }
1656                 }
1657                 else {
1658                         if (socket_num == UMA_NO_CONFIG)
1659                                 port->socket_id = 0;
1660                         else
1661                                 port->socket_id = socket_num;
1662                 }
1663         }
1664
1665         q = RTE_MAX(nb_rxq, nb_txq);
1666         if (q == 0) {
1667                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1668                 return -1;
1669         }
1670         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1671         if (nb_fwd_streams_new == nb_fwd_streams)
1672                 return 0;
1673         /* clear the old */
1674         if (fwd_streams != NULL) {
1675                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1676                         if (fwd_streams[sm_id] == NULL)
1677                                 continue;
1678                         rte_free(fwd_streams[sm_id]);
1679                         fwd_streams[sm_id] = NULL;
1680                 }
1681                 rte_free(fwd_streams);
1682                 fwd_streams = NULL;
1683         }
1684
1685         /* init new */
1686         nb_fwd_streams = nb_fwd_streams_new;
1687         if (nb_fwd_streams) {
1688                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1689                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1690                         RTE_CACHE_LINE_SIZE);
1691                 if (fwd_streams == NULL)
1692                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1693                                  " (struct fwd_stream *)) failed\n",
1694                                  nb_fwd_streams);
1695
1696                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1697                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1698                                 " struct fwd_stream", sizeof(struct fwd_stream),
1699                                 RTE_CACHE_LINE_SIZE);
1700                         if (fwd_streams[sm_id] == NULL)
1701                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1702                                          "(struct fwd_stream) failed\n");
1703                 }
1704         }
1705
1706         return 0;
1707 }
1708
1709 static void
1710 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1711 {
1712         uint64_t total_burst, sburst;
1713         uint64_t nb_burst;
1714         uint64_t burst_stats[4];
1715         uint16_t pktnb_stats[4];
1716         uint16_t nb_pkt;
1717         int burst_percent[4], sburstp;
1718         int i;
1719
1720         /*
1721          * First compute the total number of packet bursts and the
1722          * two highest numbers of bursts of the same number of packets.
1723          */
1724         memset(&burst_stats, 0x0, sizeof(burst_stats));
1725         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1726
1727         /* Show stats for 0 burst size always */
1728         total_burst = pbs->pkt_burst_spread[0];
1729         burst_stats[0] = pbs->pkt_burst_spread[0];
1730         pktnb_stats[0] = 0;
1731
1732         /* Find the next 2 burst sizes with highest occurrences. */
1733         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1734                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1735
1736                 if (nb_burst == 0)
1737                         continue;
1738
1739                 total_burst += nb_burst;
1740
1741                 if (nb_burst > burst_stats[1]) {
1742                         burst_stats[2] = burst_stats[1];
1743                         pktnb_stats[2] = pktnb_stats[1];
1744                         burst_stats[1] = nb_burst;
1745                         pktnb_stats[1] = nb_pkt;
1746                 } else if (nb_burst > burst_stats[2]) {
1747                         burst_stats[2] = nb_burst;
1748                         pktnb_stats[2] = nb_pkt;
1749                 }
1750         }
1751         if (total_burst == 0)
1752                 return;
1753
1754         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1755         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1756                 if (i == 3) {
1757                         printf("%d%% of other]\n", 100 - sburstp);
1758                         return;
1759                 }
1760
1761                 sburst += burst_stats[i];
1762                 if (sburst == total_burst) {
1763                         printf("%d%% of %d pkts]\n",
1764                                 100 - sburstp, (int) pktnb_stats[i]);
1765                         return;
1766                 }
1767
1768                 burst_percent[i] =
1769                         (double)burst_stats[i] / total_burst * 100;
1770                 printf("%d%% of %d pkts + ",
1771                         burst_percent[i], (int) pktnb_stats[i]);
1772                 sburstp += burst_percent[i];
1773         }
1774 }
1775
1776 static void
1777 fwd_stream_stats_display(streamid_t stream_id)
1778 {
1779         struct fwd_stream *fs;
1780         static const char *fwd_top_stats_border = "-------";
1781
1782         fs = fwd_streams[stream_id];
1783         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1784             (fs->fwd_dropped == 0))
1785                 return;
1786         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1787                "TX Port=%2d/Queue=%2d %s\n",
1788                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1789                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1790         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1791                " TX-dropped: %-14"PRIu64,
1792                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1793
1794         /* if checksum mode */
1795         if (cur_fwd_eng == &csum_fwd_engine) {
1796                 printf("  RX- bad IP checksum: %-14"PRIu64
1797                        "  Rx- bad L4 checksum: %-14"PRIu64
1798                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1799                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1800                         fs->rx_bad_outer_l4_csum);
1801         } else {
1802                 printf("\n");
1803         }
1804
1805         if (record_burst_stats) {
1806                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1807                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1808         }
1809 }
1810
1811 void
1812 fwd_stats_display(void)
1813 {
1814         static const char *fwd_stats_border = "----------------------";
1815         static const char *acc_stats_border = "+++++++++++++++";
1816         struct {
1817                 struct fwd_stream *rx_stream;
1818                 struct fwd_stream *tx_stream;
1819                 uint64_t tx_dropped;
1820                 uint64_t rx_bad_ip_csum;
1821                 uint64_t rx_bad_l4_csum;
1822                 uint64_t rx_bad_outer_l4_csum;
1823         } ports_stats[RTE_MAX_ETHPORTS];
1824         uint64_t total_rx_dropped = 0;
1825         uint64_t total_tx_dropped = 0;
1826         uint64_t total_rx_nombuf = 0;
1827         struct rte_eth_stats stats;
1828         uint64_t fwd_cycles = 0;
1829         uint64_t total_recv = 0;
1830         uint64_t total_xmit = 0;
1831         struct rte_port *port;
1832         streamid_t sm_id;
1833         portid_t pt_id;
1834         int i;
1835
1836         memset(ports_stats, 0, sizeof(ports_stats));
1837
1838         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1839                 struct fwd_stream *fs = fwd_streams[sm_id];
1840
1841                 if (cur_fwd_config.nb_fwd_streams >
1842                     cur_fwd_config.nb_fwd_ports) {
1843                         fwd_stream_stats_display(sm_id);
1844                 } else {
1845                         ports_stats[fs->tx_port].tx_stream = fs;
1846                         ports_stats[fs->rx_port].rx_stream = fs;
1847                 }
1848
1849                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1850
1851                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1852                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1853                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1854                                 fs->rx_bad_outer_l4_csum;
1855
1856                 if (record_core_cycles)
1857                         fwd_cycles += fs->core_cycles;
1858         }
1859         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1860                 uint8_t j;
1861
1862                 pt_id = fwd_ports_ids[i];
1863                 port = &ports[pt_id];
1864
1865                 rte_eth_stats_get(pt_id, &stats);
1866                 stats.ipackets -= port->stats.ipackets;
1867                 stats.opackets -= port->stats.opackets;
1868                 stats.ibytes -= port->stats.ibytes;
1869                 stats.obytes -= port->stats.obytes;
1870                 stats.imissed -= port->stats.imissed;
1871                 stats.oerrors -= port->stats.oerrors;
1872                 stats.rx_nombuf -= port->stats.rx_nombuf;
1873
1874                 total_recv += stats.ipackets;
1875                 total_xmit += stats.opackets;
1876                 total_rx_dropped += stats.imissed;
1877                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1878                 total_tx_dropped += stats.oerrors;
1879                 total_rx_nombuf  += stats.rx_nombuf;
1880
1881                 printf("\n  %s Forward statistics for port %-2d %s\n",
1882                        fwd_stats_border, pt_id, fwd_stats_border);
1883
1884                 if (!port->rx_queue_stats_mapping_enabled &&
1885                     !port->tx_queue_stats_mapping_enabled) {
1886                         printf("  RX-packets: %-14"PRIu64
1887                                " RX-dropped: %-14"PRIu64
1888                                "RX-total: %-"PRIu64"\n",
1889                                stats.ipackets, stats.imissed,
1890                                stats.ipackets + stats.imissed);
1891
1892                         if (cur_fwd_eng == &csum_fwd_engine)
1893                                 printf("  Bad-ipcsum: %-14"PRIu64
1894                                        " Bad-l4csum: %-14"PRIu64
1895                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1896                                        ports_stats[pt_id].rx_bad_ip_csum,
1897                                        ports_stats[pt_id].rx_bad_l4_csum,
1898                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1899                         if (stats.ierrors + stats.rx_nombuf > 0) {
1900                                 printf("  RX-error: %-"PRIu64"\n",
1901                                        stats.ierrors);
1902                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1903                                        stats.rx_nombuf);
1904                         }
1905
1906                         printf("  TX-packets: %-14"PRIu64
1907                                " TX-dropped: %-14"PRIu64
1908                                "TX-total: %-"PRIu64"\n",
1909                                stats.opackets, ports_stats[pt_id].tx_dropped,
1910                                stats.opackets + ports_stats[pt_id].tx_dropped);
1911                 } else {
1912                         printf("  RX-packets:             %14"PRIu64
1913                                "    RX-dropped:%14"PRIu64
1914                                "    RX-total:%14"PRIu64"\n",
1915                                stats.ipackets, stats.imissed,
1916                                stats.ipackets + stats.imissed);
1917
1918                         if (cur_fwd_eng == &csum_fwd_engine)
1919                                 printf("  Bad-ipcsum:%14"PRIu64
1920                                        "    Bad-l4csum:%14"PRIu64
1921                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1922                                        ports_stats[pt_id].rx_bad_ip_csum,
1923                                        ports_stats[pt_id].rx_bad_l4_csum,
1924                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1925                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1926                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1927                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1928                                        stats.rx_nombuf);
1929                         }
1930
1931                         printf("  TX-packets:             %14"PRIu64
1932                                "    TX-dropped:%14"PRIu64
1933                                "    TX-total:%14"PRIu64"\n",
1934                                stats.opackets, ports_stats[pt_id].tx_dropped,
1935                                stats.opackets + ports_stats[pt_id].tx_dropped);
1936                 }
1937
1938                 if (record_burst_stats) {
1939                         if (ports_stats[pt_id].rx_stream)
1940                                 pkt_burst_stats_display("RX",
1941                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1942                         if (ports_stats[pt_id].tx_stream)
1943                                 pkt_burst_stats_display("TX",
1944                                         &ports_stats[pt_id].tx_stream->tx_burst_stats);
1945                 }
1946
1947                 if (port->rx_queue_stats_mapping_enabled) {
1948                         printf("\n");
1949                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1950                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1951                                        "     RX-errors:%14"PRIu64
1952                                        "    RX-bytes:%14"PRIu64"\n",
1953                                        j, stats.q_ipackets[j],
1954                                        stats.q_errors[j], stats.q_ibytes[j]);
1955                         }
1956                         printf("\n");
1957                 }
1958                 if (port->tx_queue_stats_mapping_enabled) {
1959                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1960                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1961                                        "                                 TX-bytes:%14"
1962                                        PRIu64"\n",
1963                                        j, stats.q_opackets[j],
1964                                        stats.q_obytes[j]);
1965                         }
1966                 }
1967
1968                 printf("  %s--------------------------------%s\n",
1969                        fwd_stats_border, fwd_stats_border);
1970         }
1971
1972         printf("\n  %s Accumulated forward statistics for all ports"
1973                "%s\n",
1974                acc_stats_border, acc_stats_border);
1975         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1976                "%-"PRIu64"\n"
1977                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1978                "%-"PRIu64"\n",
1979                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1980                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1981         if (total_rx_nombuf > 0)
1982                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1983         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1984                "%s\n",
1985                acc_stats_border, acc_stats_border);
1986         if (record_core_cycles) {
1987 #define CYC_PER_MHZ 1E6
1988                 if (total_recv > 0 || total_xmit > 0) {
1989                         uint64_t total_pkts = 0;
1990                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1991                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1992                                 total_pkts = total_xmit;
1993                         else
1994                                 total_pkts = total_recv;
1995
1996                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1997                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1998                                " MHz Clock\n",
1999                                (double) fwd_cycles / total_pkts,
2000                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2001                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2002                 }
2003         }
2004 }
2005
2006 void
2007 fwd_stats_reset(void)
2008 {
2009         streamid_t sm_id;
2010         portid_t pt_id;
2011         int i;
2012
2013         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2014                 pt_id = fwd_ports_ids[i];
2015                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2016         }
2017         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2018                 struct fwd_stream *fs = fwd_streams[sm_id];
2019
2020                 fs->rx_packets = 0;
2021                 fs->tx_packets = 0;
2022                 fs->fwd_dropped = 0;
2023                 fs->rx_bad_ip_csum = 0;
2024                 fs->rx_bad_l4_csum = 0;
2025                 fs->rx_bad_outer_l4_csum = 0;
2026
2027                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2028                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2029                 fs->core_cycles = 0;
2030         }
2031 }
2032
2033 static void
2034 flush_fwd_rx_queues(void)
2035 {
2036         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2037         portid_t  rxp;
2038         portid_t port_id;
2039         queueid_t rxq;
2040         uint16_t  nb_rx;
2041         uint16_t  i;
2042         uint8_t   j;
2043         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2044         uint64_t timer_period;
2045
2046         /* convert to number of cycles */
2047         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2048
2049         for (j = 0; j < 2; j++) {
2050                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2051                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2052                                 port_id = fwd_ports_ids[rxp];
2053                                 /**
2054                                 * testpmd can stuck in the below do while loop
2055                                 * if rte_eth_rx_burst() always returns nonzero
2056                                 * packets. So timer is added to exit this loop
2057                                 * after 1sec timer expiry.
2058                                 */
2059                                 prev_tsc = rte_rdtsc();
2060                                 do {
2061                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2062                                                 pkts_burst, MAX_PKT_BURST);
2063                                         for (i = 0; i < nb_rx; i++)
2064                                                 rte_pktmbuf_free(pkts_burst[i]);
2065
2066                                         cur_tsc = rte_rdtsc();
2067                                         diff_tsc = cur_tsc - prev_tsc;
2068                                         timer_tsc += diff_tsc;
2069                                 } while ((nb_rx > 0) &&
2070                                         (timer_tsc < timer_period));
2071                                 timer_tsc = 0;
2072                         }
2073                 }
2074                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2075         }
2076 }
2077
2078 static void
2079 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2080 {
2081         struct fwd_stream **fsm;
2082         streamid_t nb_fs;
2083         streamid_t sm_id;
2084 #ifdef RTE_LIB_BITRATESTATS
2085         uint64_t tics_per_1sec;
2086         uint64_t tics_datum;
2087         uint64_t tics_current;
2088         uint16_t i, cnt_ports;
2089
2090         cnt_ports = nb_ports;
2091         tics_datum = rte_rdtsc();
2092         tics_per_1sec = rte_get_timer_hz();
2093 #endif
2094         fsm = &fwd_streams[fc->stream_idx];
2095         nb_fs = fc->stream_nb;
2096         do {
2097                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2098                         (*pkt_fwd)(fsm[sm_id]);
2099 #ifdef RTE_LIB_BITRATESTATS
2100                 if (bitrate_enabled != 0 &&
2101                                 bitrate_lcore_id == rte_lcore_id()) {
2102                         tics_current = rte_rdtsc();
2103                         if (tics_current - tics_datum >= tics_per_1sec) {
2104                                 /* Periodic bitrate calculation */
2105                                 for (i = 0; i < cnt_ports; i++)
2106                                         rte_stats_bitrate_calc(bitrate_data,
2107                                                 ports_ids[i]);
2108                                 tics_datum = tics_current;
2109                         }
2110                 }
2111 #endif
2112 #ifdef RTE_LIB_LATENCYSTATS
2113                 if (latencystats_enabled != 0 &&
2114                                 latencystats_lcore_id == rte_lcore_id())
2115                         rte_latencystats_update();
2116 #endif
2117
2118         } while (! fc->stopped);
2119 }
2120
2121 static int
2122 start_pkt_forward_on_core(void *fwd_arg)
2123 {
2124         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2125                              cur_fwd_config.fwd_eng->packet_fwd);
2126         return 0;
2127 }
2128
2129 /*
2130  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2131  * Used to start communication flows in network loopback test configurations.
2132  */
2133 static int
2134 run_one_txonly_burst_on_core(void *fwd_arg)
2135 {
2136         struct fwd_lcore *fwd_lc;
2137         struct fwd_lcore tmp_lcore;
2138
2139         fwd_lc = (struct fwd_lcore *) fwd_arg;
2140         tmp_lcore = *fwd_lc;
2141         tmp_lcore.stopped = 1;
2142         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2143         return 0;
2144 }
2145
2146 /*
2147  * Launch packet forwarding:
2148  *     - Setup per-port forwarding context.
2149  *     - launch logical cores with their forwarding configuration.
2150  */
2151 static void
2152 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2153 {
2154         port_fwd_begin_t port_fwd_begin;
2155         unsigned int i;
2156         unsigned int lc_id;
2157         int diag;
2158
2159         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2160         if (port_fwd_begin != NULL) {
2161                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2162                         (*port_fwd_begin)(fwd_ports_ids[i]);
2163         }
2164         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2165                 lc_id = fwd_lcores_cpuids[i];
2166                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2167                         fwd_lcores[i]->stopped = 0;
2168                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2169                                                      fwd_lcores[i], lc_id);
2170                         if (diag != 0)
2171                                 printf("launch lcore %u failed - diag=%d\n",
2172                                        lc_id, diag);
2173                 }
2174         }
2175 }
2176
2177 /*
2178  * Launch packet forwarding configuration.
2179  */
2180 void
2181 start_packet_forwarding(int with_tx_first)
2182 {
2183         port_fwd_begin_t port_fwd_begin;
2184         port_fwd_end_t  port_fwd_end;
2185         struct rte_port *port;
2186         unsigned int i;
2187         portid_t   pt_id;
2188
2189         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2190                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2191
2192         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2193                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2194
2195         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2196                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2197                 (!nb_rxq || !nb_txq))
2198                 rte_exit(EXIT_FAILURE,
2199                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2200                         cur_fwd_eng->fwd_mode_name);
2201
2202         if (all_ports_started() == 0) {
2203                 printf("Not all ports were started\n");
2204                 return;
2205         }
2206         if (test_done == 0) {
2207                 printf("Packet forwarding already started\n");
2208                 return;
2209         }
2210
2211
2212         if(dcb_test) {
2213                 for (i = 0; i < nb_fwd_ports; i++) {
2214                         pt_id = fwd_ports_ids[i];
2215                         port = &ports[pt_id];
2216                         if (!port->dcb_flag) {
2217                                 printf("In DCB mode, all forwarding ports must "
2218                                        "be configured in this mode.\n");
2219                                 return;
2220                         }
2221                 }
2222                 if (nb_fwd_lcores == 1) {
2223                         printf("In DCB mode,the nb forwarding cores "
2224                                "should be larger than 1.\n");
2225                         return;
2226                 }
2227         }
2228         test_done = 0;
2229
2230         fwd_config_setup();
2231
2232         if(!no_flush_rx)
2233                 flush_fwd_rx_queues();
2234
2235         pkt_fwd_config_display(&cur_fwd_config);
2236         rxtx_config_display();
2237
2238         fwd_stats_reset();
2239         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2240                 pt_id = fwd_ports_ids[i];
2241                 port = &ports[pt_id];
2242                 map_port_queue_stats_mapping_registers(pt_id, port);
2243         }
2244         if (with_tx_first) {
2245                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2246                 if (port_fwd_begin != NULL) {
2247                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2248                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2249                 }
2250                 while (with_tx_first--) {
2251                         launch_packet_forwarding(
2252                                         run_one_txonly_burst_on_core);
2253                         rte_eal_mp_wait_lcore();
2254                 }
2255                 port_fwd_end = tx_only_engine.port_fwd_end;
2256                 if (port_fwd_end != NULL) {
2257                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2258                                 (*port_fwd_end)(fwd_ports_ids[i]);
2259                 }
2260         }
2261         launch_packet_forwarding(start_pkt_forward_on_core);
2262 }
2263
2264 void
2265 stop_packet_forwarding(void)
2266 {
2267         port_fwd_end_t port_fwd_end;
2268         lcoreid_t lc_id;
2269         portid_t pt_id;
2270         int i;
2271
2272         if (test_done) {
2273                 printf("Packet forwarding not started\n");
2274                 return;
2275         }
2276         printf("Telling cores to stop...");
2277         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2278                 fwd_lcores[lc_id]->stopped = 1;
2279         printf("\nWaiting for lcores to finish...\n");
2280         rte_eal_mp_wait_lcore();
2281         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2282         if (port_fwd_end != NULL) {
2283                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2284                         pt_id = fwd_ports_ids[i];
2285                         (*port_fwd_end)(pt_id);
2286                 }
2287         }
2288
2289         fwd_stats_display();
2290
2291         printf("\nDone.\n");
2292         test_done = 1;
2293 }
2294
2295 void
2296 dev_set_link_up(portid_t pid)
2297 {
2298         if (rte_eth_dev_set_link_up(pid) < 0)
2299                 printf("\nSet link up fail.\n");
2300 }
2301
2302 void
2303 dev_set_link_down(portid_t pid)
2304 {
2305         if (rte_eth_dev_set_link_down(pid) < 0)
2306                 printf("\nSet link down fail.\n");
2307 }
2308
2309 static int
2310 all_ports_started(void)
2311 {
2312         portid_t pi;
2313         struct rte_port *port;
2314
2315         RTE_ETH_FOREACH_DEV(pi) {
2316                 port = &ports[pi];
2317                 /* Check if there is a port which is not started */
2318                 if ((port->port_status != RTE_PORT_STARTED) &&
2319                         (port->slave_flag == 0))
2320                         return 0;
2321         }
2322
2323         /* No port is not started */
2324         return 1;
2325 }
2326
2327 int
2328 port_is_stopped(portid_t port_id)
2329 {
2330         struct rte_port *port = &ports[port_id];
2331
2332         if ((port->port_status != RTE_PORT_STOPPED) &&
2333             (port->slave_flag == 0))
2334                 return 0;
2335         return 1;
2336 }
2337
2338 int
2339 all_ports_stopped(void)
2340 {
2341         portid_t pi;
2342
2343         RTE_ETH_FOREACH_DEV(pi) {
2344                 if (!port_is_stopped(pi))
2345                         return 0;
2346         }
2347
2348         return 1;
2349 }
2350
2351 int
2352 port_is_started(portid_t port_id)
2353 {
2354         if (port_id_is_invalid(port_id, ENABLED_WARN))
2355                 return 0;
2356
2357         if (ports[port_id].port_status != RTE_PORT_STARTED)
2358                 return 0;
2359
2360         return 1;
2361 }
2362
2363 /* Configure the Rx and Tx hairpin queues for the selected port. */
2364 static int
2365 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2366 {
2367         queueid_t qi;
2368         struct rte_eth_hairpin_conf hairpin_conf = {
2369                 .peer_count = 1,
2370         };
2371         int i;
2372         int diag;
2373         struct rte_port *port = &ports[pi];
2374         uint16_t peer_rx_port = pi;
2375         uint16_t peer_tx_port = pi;
2376         uint32_t manual = 1;
2377         uint32_t tx_exp = hairpin_mode & 0x10;
2378
2379         if (!(hairpin_mode & 0xf)) {
2380                 peer_rx_port = pi;
2381                 peer_tx_port = pi;
2382                 manual = 0;
2383         } else if (hairpin_mode & 0x1) {
2384                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2385                                                        RTE_ETH_DEV_NO_OWNER);
2386                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2387                         peer_tx_port = rte_eth_find_next_owned_by(0,
2388                                                 RTE_ETH_DEV_NO_OWNER);
2389                 if (p_pi != RTE_MAX_ETHPORTS) {
2390                         peer_rx_port = p_pi;
2391                 } else {
2392                         uint16_t next_pi;
2393
2394                         /* Last port will be the peer RX port of the first. */
2395                         RTE_ETH_FOREACH_DEV(next_pi)
2396                                 peer_rx_port = next_pi;
2397                 }
2398                 manual = 1;
2399         } else if (hairpin_mode & 0x2) {
2400                 if (cnt_pi & 0x1) {
2401                         peer_rx_port = p_pi;
2402                 } else {
2403                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2404                                                 RTE_ETH_DEV_NO_OWNER);
2405                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2406                                 peer_rx_port = pi;
2407                 }
2408                 peer_tx_port = peer_rx_port;
2409                 manual = 1;
2410         }
2411
2412         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2413                 hairpin_conf.peers[0].port = peer_rx_port;
2414                 hairpin_conf.peers[0].queue = i + nb_rxq;
2415                 hairpin_conf.manual_bind = !!manual;
2416                 hairpin_conf.tx_explicit = !!tx_exp;
2417                 diag = rte_eth_tx_hairpin_queue_setup
2418                         (pi, qi, nb_txd, &hairpin_conf);
2419                 i++;
2420                 if (diag == 0)
2421                         continue;
2422
2423                 /* Fail to setup rx queue, return */
2424                 if (rte_atomic16_cmpset(&(port->port_status),
2425                                         RTE_PORT_HANDLING,
2426                                         RTE_PORT_STOPPED) == 0)
2427                         printf("Port %d can not be set back "
2428                                         "to stopped\n", pi);
2429                 printf("Fail to configure port %d hairpin "
2430                                 "queues\n", pi);
2431                 /* try to reconfigure queues next time */
2432                 port->need_reconfig_queues = 1;
2433                 return -1;
2434         }
2435         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2436                 hairpin_conf.peers[0].port = peer_tx_port;
2437                 hairpin_conf.peers[0].queue = i + nb_txq;
2438                 hairpin_conf.manual_bind = !!manual;
2439                 hairpin_conf.tx_explicit = !!tx_exp;
2440                 diag = rte_eth_rx_hairpin_queue_setup
2441                         (pi, qi, nb_rxd, &hairpin_conf);
2442                 i++;
2443                 if (diag == 0)
2444                         continue;
2445
2446                 /* Fail to setup rx queue, return */
2447                 if (rte_atomic16_cmpset(&(port->port_status),
2448                                         RTE_PORT_HANDLING,
2449                                         RTE_PORT_STOPPED) == 0)
2450                         printf("Port %d can not be set back "
2451                                         "to stopped\n", pi);
2452                 printf("Fail to configure port %d hairpin "
2453                                 "queues\n", pi);
2454                 /* try to reconfigure queues next time */
2455                 port->need_reconfig_queues = 1;
2456                 return -1;
2457         }
2458         return 0;
2459 }
2460
2461 /* Configure the Rx with optional split. */
2462 int
2463 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2464                uint16_t nb_rx_desc, unsigned int socket_id,
2465                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2466 {
2467         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2468         unsigned int i, mp_n;
2469         int ret;
2470
2471         if (rx_pkt_nb_segs <= 1 ||
2472             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2473                 rx_conf->rx_seg = NULL;
2474                 rx_conf->rx_nseg = 0;
2475                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2476                                              nb_rx_desc, socket_id,
2477                                              rx_conf, mp);
2478                 return ret;
2479         }
2480         for (i = 0; i < rx_pkt_nb_segs; i++) {
2481                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2482                 struct rte_mempool *mpx;
2483                 /*
2484                  * Use last valid pool for the segments with number
2485                  * exceeding the pool index.
2486                  */
2487                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2488                 mpx = mbuf_pool_find(socket_id, mp_n);
2489                 /* Handle zero as mbuf data buffer size. */
2490                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2491                                    rx_pkt_seg_lengths[i] :
2492                                    mbuf_data_size[mp_n];
2493                 rx_seg->offset = i < rx_pkt_nb_offs ?
2494                                    rx_pkt_seg_offsets[i] : 0;
2495                 rx_seg->mp = mpx ? mpx : mp;
2496         }
2497         rx_conf->rx_nseg = rx_pkt_nb_segs;
2498         rx_conf->rx_seg = rx_useg;
2499         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2500                                     socket_id, rx_conf, NULL);
2501         rx_conf->rx_seg = NULL;
2502         rx_conf->rx_nseg = 0;
2503         return ret;
2504 }
2505
2506 int
2507 start_port(portid_t pid)
2508 {
2509         int diag, need_check_link_status = -1;
2510         portid_t pi;
2511         portid_t p_pi = RTE_MAX_ETHPORTS;
2512         portid_t pl[RTE_MAX_ETHPORTS];
2513         portid_t peer_pl[RTE_MAX_ETHPORTS];
2514         uint16_t cnt_pi = 0;
2515         uint16_t cfg_pi = 0;
2516         int peer_pi;
2517         queueid_t qi;
2518         struct rte_port *port;
2519         struct rte_ether_addr mac_addr;
2520         struct rte_eth_hairpin_cap cap;
2521
2522         if (port_id_is_invalid(pid, ENABLED_WARN))
2523                 return 0;
2524
2525         if(dcb_config)
2526                 dcb_test = 1;
2527         RTE_ETH_FOREACH_DEV(pi) {
2528                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2529                         continue;
2530
2531                 need_check_link_status = 0;
2532                 port = &ports[pi];
2533                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2534                                                  RTE_PORT_HANDLING) == 0) {
2535                         printf("Port %d is now not stopped\n", pi);
2536                         continue;
2537                 }
2538
2539                 if (port->need_reconfig > 0) {
2540                         port->need_reconfig = 0;
2541
2542                         if (flow_isolate_all) {
2543                                 int ret = port_flow_isolate(pi, 1);
2544                                 if (ret) {
2545                                         printf("Failed to apply isolated"
2546                                                " mode on port %d\n", pi);
2547                                         return -1;
2548                                 }
2549                         }
2550                         configure_rxtx_dump_callbacks(0);
2551                         printf("Configuring Port %d (socket %u)\n", pi,
2552                                         port->socket_id);
2553                         if (nb_hairpinq > 0 &&
2554                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2555                                 printf("Port %d doesn't support hairpin "
2556                                        "queues\n", pi);
2557                                 return -1;
2558                         }
2559                         /* configure port */
2560                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2561                                                      nb_txq + nb_hairpinq,
2562                                                      &(port->dev_conf));
2563                         if (diag != 0) {
2564                                 if (rte_atomic16_cmpset(&(port->port_status),
2565                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2566                                         printf("Port %d can not be set back "
2567                                                         "to stopped\n", pi);
2568                                 printf("Fail to configure port %d\n", pi);
2569                                 /* try to reconfigure port next time */
2570                                 port->need_reconfig = 1;
2571                                 return -1;
2572                         }
2573                 }
2574                 if (port->need_reconfig_queues > 0) {
2575                         port->need_reconfig_queues = 0;
2576                         /* setup tx queues */
2577                         for (qi = 0; qi < nb_txq; qi++) {
2578                                 if ((numa_support) &&
2579                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2580                                         diag = rte_eth_tx_queue_setup(pi, qi,
2581                                                 port->nb_tx_desc[qi],
2582                                                 txring_numa[pi],
2583                                                 &(port->tx_conf[qi]));
2584                                 else
2585                                         diag = rte_eth_tx_queue_setup(pi, qi,
2586                                                 port->nb_tx_desc[qi],
2587                                                 port->socket_id,
2588                                                 &(port->tx_conf[qi]));
2589
2590                                 if (diag == 0)
2591                                         continue;
2592
2593                                 /* Fail to setup tx queue, return */
2594                                 if (rte_atomic16_cmpset(&(port->port_status),
2595                                                         RTE_PORT_HANDLING,
2596                                                         RTE_PORT_STOPPED) == 0)
2597                                         printf("Port %d can not be set back "
2598                                                         "to stopped\n", pi);
2599                                 printf("Fail to configure port %d tx queues\n",
2600                                        pi);
2601                                 /* try to reconfigure queues next time */
2602                                 port->need_reconfig_queues = 1;
2603                                 return -1;
2604                         }
2605                         for (qi = 0; qi < nb_rxq; qi++) {
2606                                 /* setup rx queues */
2607                                 if ((numa_support) &&
2608                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2609                                         struct rte_mempool * mp =
2610                                                 mbuf_pool_find
2611                                                         (rxring_numa[pi], 0);
2612                                         if (mp == NULL) {
2613                                                 printf("Failed to setup RX queue:"
2614                                                         "No mempool allocation"
2615                                                         " on the socket %d\n",
2616                                                         rxring_numa[pi]);
2617                                                 return -1;
2618                                         }
2619
2620                                         diag = rx_queue_setup(pi, qi,
2621                                              port->nb_rx_desc[qi],
2622                                              rxring_numa[pi],
2623                                              &(port->rx_conf[qi]),
2624                                              mp);
2625                                 } else {
2626                                         struct rte_mempool *mp =
2627                                                 mbuf_pool_find
2628                                                         (port->socket_id, 0);
2629                                         if (mp == NULL) {
2630                                                 printf("Failed to setup RX queue:"
2631                                                         "No mempool allocation"
2632                                                         " on the socket %d\n",
2633                                                         port->socket_id);
2634                                                 return -1;
2635                                         }
2636                                         diag = rx_queue_setup(pi, qi,
2637                                              port->nb_rx_desc[qi],
2638                                              port->socket_id,
2639                                              &(port->rx_conf[qi]),
2640                                              mp);
2641                                 }
2642                                 if (diag == 0)
2643                                         continue;
2644
2645                                 /* Fail to setup rx queue, return */
2646                                 if (rte_atomic16_cmpset(&(port->port_status),
2647                                                         RTE_PORT_HANDLING,
2648                                                         RTE_PORT_STOPPED) == 0)
2649                                         printf("Port %d can not be set back "
2650                                                         "to stopped\n", pi);
2651                                 printf("Fail to configure port %d rx queues\n",
2652                                        pi);
2653                                 /* try to reconfigure queues next time */
2654                                 port->need_reconfig_queues = 1;
2655                                 return -1;
2656                         }
2657                         /* setup hairpin queues */
2658                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2659                                 return -1;
2660                 }
2661                 configure_rxtx_dump_callbacks(verbose_level);
2662                 if (clear_ptypes) {
2663                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2664                                         NULL, 0);
2665                         if (diag < 0)
2666                                 printf(
2667                                 "Port %d: Failed to disable Ptype parsing\n",
2668                                 pi);
2669                 }
2670
2671                 p_pi = pi;
2672                 cnt_pi++;
2673
2674                 /* start port */
2675                 if (rte_eth_dev_start(pi) < 0) {
2676                         printf("Fail to start port %d\n", pi);
2677
2678                         /* Fail to setup rx queue, return */
2679                         if (rte_atomic16_cmpset(&(port->port_status),
2680                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2681                                 printf("Port %d can not be set back to "
2682                                                         "stopped\n", pi);
2683                         continue;
2684                 }
2685
2686                 if (rte_atomic16_cmpset(&(port->port_status),
2687                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2688                         printf("Port %d can not be set into started\n", pi);
2689
2690                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2691                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2692                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2693                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2694                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2695
2696                 /* at least one port started, need checking link status */
2697                 need_check_link_status = 1;
2698
2699                 pl[cfg_pi++] = pi;
2700         }
2701
2702         if (need_check_link_status == 1 && !no_link_check)
2703                 check_all_ports_link_status(RTE_PORT_ALL);
2704         else if (need_check_link_status == 0)
2705                 printf("Please stop the ports first\n");
2706
2707         if (hairpin_mode & 0xf) {
2708                 uint16_t i;
2709                 int j;
2710
2711                 /* bind all started hairpin ports */
2712                 for (i = 0; i < cfg_pi; i++) {
2713                         pi = pl[i];
2714                         /* bind current Tx to all peer Rx */
2715                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2716                                                         RTE_MAX_ETHPORTS, 1);
2717                         if (peer_pi < 0)
2718                                 return peer_pi;
2719                         for (j = 0; j < peer_pi; j++) {
2720                                 if (!port_is_started(peer_pl[j]))
2721                                         continue;
2722                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2723                                 if (diag < 0) {
2724                                         printf("Error during binding hairpin"
2725                                                " Tx port %u to %u: %s\n",
2726                                                pi, peer_pl[j],
2727                                                rte_strerror(-diag));
2728                                         return -1;
2729                                 }
2730                         }
2731                         /* bind all peer Tx to current Rx */
2732                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2733                                                         RTE_MAX_ETHPORTS, 0);
2734                         if (peer_pi < 0)
2735                                 return peer_pi;
2736                         for (j = 0; j < peer_pi; j++) {
2737                                 if (!port_is_started(peer_pl[j]))
2738                                         continue;
2739                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2740                                 if (diag < 0) {
2741                                         printf("Error during binding hairpin"
2742                                                " Tx port %u to %u: %s\n",
2743                                                peer_pl[j], pi,
2744                                                rte_strerror(-diag));
2745                                         return -1;
2746                                 }
2747                         }
2748                 }
2749         }
2750
2751         printf("Done\n");
2752         return 0;
2753 }
2754
2755 void
2756 stop_port(portid_t pid)
2757 {
2758         portid_t pi;
2759         struct rte_port *port;
2760         int need_check_link_status = 0;
2761         portid_t peer_pl[RTE_MAX_ETHPORTS];
2762         int peer_pi;
2763
2764         if (dcb_test) {
2765                 dcb_test = 0;
2766                 dcb_config = 0;
2767         }
2768
2769         if (port_id_is_invalid(pid, ENABLED_WARN))
2770                 return;
2771
2772         printf("Stopping ports...\n");
2773
2774         RTE_ETH_FOREACH_DEV(pi) {
2775                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2776                         continue;
2777
2778                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2779                         printf("Please remove port %d from forwarding configuration.\n", pi);
2780                         continue;
2781                 }
2782
2783                 if (port_is_bonding_slave(pi)) {
2784                         printf("Please remove port %d from bonded device.\n", pi);
2785                         continue;
2786                 }
2787
2788                 port = &ports[pi];
2789                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2790                                                 RTE_PORT_HANDLING) == 0)
2791                         continue;
2792
2793                 if (hairpin_mode & 0xf) {
2794                         int j;
2795
2796                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2797                         /* unbind all peer Tx from current Rx */
2798                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2799                                                         RTE_MAX_ETHPORTS, 0);
2800                         if (peer_pi < 0)
2801                                 continue;
2802                         for (j = 0; j < peer_pi; j++) {
2803                                 if (!port_is_started(peer_pl[j]))
2804                                         continue;
2805                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2806                         }
2807                 }
2808
2809                 if (rte_eth_dev_stop(pi) != 0)
2810                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2811                                 pi);
2812
2813                 if (rte_atomic16_cmpset(&(port->port_status),
2814                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2815                         printf("Port %d can not be set into stopped\n", pi);
2816                 need_check_link_status = 1;
2817         }
2818         if (need_check_link_status && !no_link_check)
2819                 check_all_ports_link_status(RTE_PORT_ALL);
2820
2821         printf("Done\n");
2822 }
2823
2824 static void
2825 remove_invalid_ports_in(portid_t *array, portid_t *total)
2826 {
2827         portid_t i;
2828         portid_t new_total = 0;
2829
2830         for (i = 0; i < *total; i++)
2831                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2832                         array[new_total] = array[i];
2833                         new_total++;
2834                 }
2835         *total = new_total;
2836 }
2837
2838 static void
2839 remove_invalid_ports(void)
2840 {
2841         remove_invalid_ports_in(ports_ids, &nb_ports);
2842         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2843         nb_cfg_ports = nb_fwd_ports;
2844 }
2845
2846 void
2847 close_port(portid_t pid)
2848 {
2849         portid_t pi;
2850         struct rte_port *port;
2851
2852         if (port_id_is_invalid(pid, ENABLED_WARN))
2853                 return;
2854
2855         printf("Closing ports...\n");
2856
2857         RTE_ETH_FOREACH_DEV(pi) {
2858                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2859                         continue;
2860
2861                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2862                         printf("Please remove port %d from forwarding configuration.\n", pi);
2863                         continue;
2864                 }
2865
2866                 if (port_is_bonding_slave(pi)) {
2867                         printf("Please remove port %d from bonded device.\n", pi);
2868                         continue;
2869                 }
2870
2871                 port = &ports[pi];
2872                 if (rte_atomic16_cmpset(&(port->port_status),
2873                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2874                         printf("Port %d is already closed\n", pi);
2875                         continue;
2876                 }
2877
2878                 port_flow_flush(pi);
2879                 rte_eth_dev_close(pi);
2880         }
2881
2882         remove_invalid_ports();
2883         printf("Done\n");
2884 }
2885
2886 void
2887 reset_port(portid_t pid)
2888 {
2889         int diag;
2890         portid_t pi;
2891         struct rte_port *port;
2892
2893         if (port_id_is_invalid(pid, ENABLED_WARN))
2894                 return;
2895
2896         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2897                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2898                 printf("Can not reset port(s), please stop port(s) first.\n");
2899                 return;
2900         }
2901
2902         printf("Resetting ports...\n");
2903
2904         RTE_ETH_FOREACH_DEV(pi) {
2905                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2906                         continue;
2907
2908                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2909                         printf("Please remove port %d from forwarding "
2910                                "configuration.\n", pi);
2911                         continue;
2912                 }
2913
2914                 if (port_is_bonding_slave(pi)) {
2915                         printf("Please remove port %d from bonded device.\n",
2916                                pi);
2917                         continue;
2918                 }
2919
2920                 diag = rte_eth_dev_reset(pi);
2921                 if (diag == 0) {
2922                         port = &ports[pi];
2923                         port->need_reconfig = 1;
2924                         port->need_reconfig_queues = 1;
2925                 } else {
2926                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2927                 }
2928         }
2929
2930         printf("Done\n");
2931 }
2932
2933 void
2934 attach_port(char *identifier)
2935 {
2936         portid_t pi;
2937         struct rte_dev_iterator iterator;
2938
2939         printf("Attaching a new port...\n");
2940
2941         if (identifier == NULL) {
2942                 printf("Invalid parameters are specified\n");
2943                 return;
2944         }
2945
2946         if (rte_dev_probe(identifier) < 0) {
2947                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2948                 return;
2949         }
2950
2951         /* first attach mode: event */
2952         if (setup_on_probe_event) {
2953                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2954                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2955                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2956                                         ports[pi].need_setup != 0)
2957                                 setup_attached_port(pi);
2958                 return;
2959         }
2960
2961         /* second attach mode: iterator */
2962         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2963                 /* setup ports matching the devargs used for probing */
2964                 if (port_is_forwarding(pi))
2965                         continue; /* port was already attached before */
2966                 setup_attached_port(pi);
2967         }
2968 }
2969
2970 static void
2971 setup_attached_port(portid_t pi)
2972 {
2973         unsigned int socket_id;
2974         int ret;
2975
2976         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2977         /* if socket_id is invalid, set to the first available socket. */
2978         if (check_socket_id(socket_id) < 0)
2979                 socket_id = socket_ids[0];
2980         reconfig(pi, socket_id);
2981         ret = rte_eth_promiscuous_enable(pi);
2982         if (ret != 0)
2983                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2984                         pi, rte_strerror(-ret));
2985
2986         ports_ids[nb_ports++] = pi;
2987         fwd_ports_ids[nb_fwd_ports++] = pi;
2988         nb_cfg_ports = nb_fwd_ports;
2989         ports[pi].need_setup = 0;
2990         ports[pi].port_status = RTE_PORT_STOPPED;
2991
2992         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2993         printf("Done\n");
2994 }
2995
2996 static void
2997 detach_device(struct rte_device *dev)
2998 {
2999         portid_t sibling;
3000
3001         if (dev == NULL) {
3002                 printf("Device already removed\n");
3003                 return;
3004         }
3005
3006         printf("Removing a device...\n");
3007
3008         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3009                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3010                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3011                                 printf("Port %u not stopped\n", sibling);
3012                                 return;
3013                         }
3014                         port_flow_flush(sibling);
3015                 }
3016         }
3017
3018         if (rte_dev_remove(dev) < 0) {
3019                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3020                 return;
3021         }
3022         remove_invalid_ports();
3023
3024         printf("Device is detached\n");
3025         printf("Now total ports is %d\n", nb_ports);
3026         printf("Done\n");
3027         return;
3028 }
3029
3030 void
3031 detach_port_device(portid_t port_id)
3032 {
3033         if (port_id_is_invalid(port_id, ENABLED_WARN))
3034                 return;
3035
3036         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3037                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3038                         printf("Port not stopped\n");
3039                         return;
3040                 }
3041                 printf("Port was not closed\n");
3042         }
3043
3044         detach_device(rte_eth_devices[port_id].device);
3045 }
3046
3047 void
3048 detach_devargs(char *identifier)
3049 {
3050         struct rte_dev_iterator iterator;
3051         struct rte_devargs da;
3052         portid_t port_id;
3053
3054         printf("Removing a device...\n");
3055
3056         memset(&da, 0, sizeof(da));
3057         if (rte_devargs_parsef(&da, "%s", identifier)) {
3058                 printf("cannot parse identifier\n");
3059                 if (da.args)
3060                         free(da.args);
3061                 return;
3062         }
3063
3064         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3065                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3066                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3067                                 printf("Port %u not stopped\n", port_id);
3068                                 rte_eth_iterator_cleanup(&iterator);
3069                                 return;
3070                         }
3071                         port_flow_flush(port_id);
3072                 }
3073         }
3074
3075         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3076                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3077                             da.name, da.bus->name);
3078                 return;
3079         }
3080
3081         remove_invalid_ports();
3082
3083         printf("Device %s is detached\n", identifier);
3084         printf("Now total ports is %d\n", nb_ports);
3085         printf("Done\n");
3086 }
3087
3088 void
3089 pmd_test_exit(void)
3090 {
3091         portid_t pt_id;
3092         unsigned int i;
3093         int ret;
3094
3095         if (test_done == 0)
3096                 stop_packet_forwarding();
3097
3098         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3099                 if (mempools[i]) {
3100                         if (mp_alloc_type == MP_ALLOC_ANON)
3101                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3102                                                      NULL);
3103                 }
3104         }
3105         if (ports != NULL) {
3106                 no_link_check = 1;
3107                 RTE_ETH_FOREACH_DEV(pt_id) {
3108                         printf("\nStopping port %d...\n", pt_id);
3109                         fflush(stdout);
3110                         stop_port(pt_id);
3111                 }
3112                 RTE_ETH_FOREACH_DEV(pt_id) {
3113                         printf("\nShutting down port %d...\n", pt_id);
3114                         fflush(stdout);
3115                         close_port(pt_id);
3116                 }
3117         }
3118
3119         if (hot_plug) {
3120                 ret = rte_dev_event_monitor_stop();
3121                 if (ret) {
3122                         RTE_LOG(ERR, EAL,
3123                                 "fail to stop device event monitor.");
3124                         return;
3125                 }
3126
3127                 ret = rte_dev_event_callback_unregister(NULL,
3128                         dev_event_callback, NULL);
3129                 if (ret < 0) {
3130                         RTE_LOG(ERR, EAL,
3131                                 "fail to unregister device event callback.\n");
3132                         return;
3133                 }
3134
3135                 ret = rte_dev_hotplug_handle_disable();
3136                 if (ret) {
3137                         RTE_LOG(ERR, EAL,
3138                                 "fail to disable hotplug handling.\n");
3139                         return;
3140                 }
3141         }
3142         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3143                 if (mempools[i])
3144                         rte_mempool_free(mempools[i]);
3145         }
3146
3147         printf("\nBye...\n");
3148 }
3149
3150 typedef void (*cmd_func_t)(void);
3151 struct pmd_test_command {
3152         const char *cmd_name;
3153         cmd_func_t cmd_func;
3154 };
3155
3156 /* Check the link status of all ports in up to 9s, and print them finally */
3157 static void
3158 check_all_ports_link_status(uint32_t port_mask)
3159 {
3160 #define CHECK_INTERVAL 100 /* 100ms */
3161 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3162         portid_t portid;
3163         uint8_t count, all_ports_up, print_flag = 0;
3164         struct rte_eth_link link;
3165         int ret;
3166         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3167
3168         printf("Checking link statuses...\n");
3169         fflush(stdout);
3170         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3171                 all_ports_up = 1;
3172                 RTE_ETH_FOREACH_DEV(portid) {
3173                         if ((port_mask & (1 << portid)) == 0)
3174                                 continue;
3175                         memset(&link, 0, sizeof(link));
3176                         ret = rte_eth_link_get_nowait(portid, &link);
3177                         if (ret < 0) {
3178                                 all_ports_up = 0;
3179                                 if (print_flag == 1)
3180                                         printf("Port %u link get failed: %s\n",
3181                                                 portid, rte_strerror(-ret));
3182                                 continue;
3183                         }
3184                         /* print link status if flag set */
3185                         if (print_flag == 1) {
3186                                 rte_eth_link_to_str(link_status,
3187                                         sizeof(link_status), &link);
3188                                 printf("Port %d %s\n", portid, link_status);
3189                                 continue;
3190                         }
3191                         /* clear all_ports_up flag if any link down */
3192                         if (link.link_status == ETH_LINK_DOWN) {
3193                                 all_ports_up = 0;
3194                                 break;
3195                         }
3196                 }
3197                 /* after finally printing all link status, get out */
3198                 if (print_flag == 1)
3199                         break;
3200
3201                 if (all_ports_up == 0) {
3202                         fflush(stdout);
3203                         rte_delay_ms(CHECK_INTERVAL);
3204                 }
3205
3206                 /* set the print_flag if all ports up or timeout */
3207                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3208                         print_flag = 1;
3209                 }
3210
3211                 if (lsc_interrupt)
3212                         break;
3213         }
3214 }
3215
3216 static void
3217 rmv_port_callback(void *arg)
3218 {
3219         int need_to_start = 0;
3220         int org_no_link_check = no_link_check;
3221         portid_t port_id = (intptr_t)arg;
3222         struct rte_device *dev;
3223
3224         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3225
3226         if (!test_done && port_is_forwarding(port_id)) {
3227                 need_to_start = 1;
3228                 stop_packet_forwarding();
3229         }
3230         no_link_check = 1;
3231         stop_port(port_id);
3232         no_link_check = org_no_link_check;
3233
3234         /* Save rte_device pointer before closing ethdev port */
3235         dev = rte_eth_devices[port_id].device;
3236         close_port(port_id);
3237         detach_device(dev); /* might be already removed or have more ports */
3238
3239         if (need_to_start)
3240                 start_packet_forwarding(0);
3241 }
3242
3243 /* This function is used by the interrupt thread */
3244 static int
3245 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3246                   void *ret_param)
3247 {
3248         RTE_SET_USED(param);
3249         RTE_SET_USED(ret_param);
3250
3251         if (type >= RTE_ETH_EVENT_MAX) {
3252                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3253                         port_id, __func__, type);
3254                 fflush(stderr);
3255         } else if (event_print_mask & (UINT32_C(1) << type)) {
3256                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3257                         eth_event_desc[type]);
3258                 fflush(stdout);
3259         }
3260
3261         switch (type) {
3262         case RTE_ETH_EVENT_NEW:
3263                 ports[port_id].need_setup = 1;
3264                 ports[port_id].port_status = RTE_PORT_HANDLING;
3265                 break;
3266         case RTE_ETH_EVENT_INTR_RMV:
3267                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3268                         break;
3269                 if (rte_eal_alarm_set(100000,
3270                                 rmv_port_callback, (void *)(intptr_t)port_id))
3271                         fprintf(stderr, "Could not set up deferred device removal\n");
3272                 break;
3273         case RTE_ETH_EVENT_DESTROY:
3274                 ports[port_id].port_status = RTE_PORT_CLOSED;
3275                 printf("Port %u is closed\n", port_id);
3276                 break;
3277         default:
3278                 break;
3279         }
3280         return 0;
3281 }
3282
3283 static int
3284 register_eth_event_callback(void)
3285 {
3286         int ret;
3287         enum rte_eth_event_type event;
3288
3289         for (event = RTE_ETH_EVENT_UNKNOWN;
3290                         event < RTE_ETH_EVENT_MAX; event++) {
3291                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3292                                 event,
3293                                 eth_event_callback,
3294                                 NULL);
3295                 if (ret != 0) {
3296                         TESTPMD_LOG(ERR, "Failed to register callback for "
3297                                         "%s event\n", eth_event_desc[event]);
3298                         return -1;
3299                 }
3300         }
3301
3302         return 0;
3303 }
3304
3305 /* This function is used by the interrupt thread */
3306 static void
3307 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3308                              __rte_unused void *arg)
3309 {
3310         uint16_t port_id;
3311         int ret;
3312
3313         if (type >= RTE_DEV_EVENT_MAX) {
3314                 fprintf(stderr, "%s called upon invalid event %d\n",
3315                         __func__, type);
3316                 fflush(stderr);
3317         }
3318
3319         switch (type) {
3320         case RTE_DEV_EVENT_REMOVE:
3321                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3322                         device_name);
3323                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3324                 if (ret) {
3325                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3326                                 device_name);
3327                         return;
3328                 }
3329                 /*
3330                  * Because the user's callback is invoked in eal interrupt
3331                  * callback, the interrupt callback need to be finished before
3332                  * it can be unregistered when detaching device. So finish
3333                  * callback soon and use a deferred removal to detach device
3334                  * is need. It is a workaround, once the device detaching be
3335                  * moved into the eal in the future, the deferred removal could
3336                  * be deleted.
3337                  */
3338                 if (rte_eal_alarm_set(100000,
3339                                 rmv_port_callback, (void *)(intptr_t)port_id))
3340                         RTE_LOG(ERR, EAL,
3341                                 "Could not set up deferred device removal\n");
3342                 break;
3343         case RTE_DEV_EVENT_ADD:
3344                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3345                         device_name);
3346                 /* TODO: After finish kernel driver binding,
3347                  * begin to attach port.
3348                  */
3349                 break;
3350         default:
3351                 break;
3352         }
3353 }
3354
3355 static int
3356 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3357 {
3358         uint16_t i;
3359         int diag;
3360         uint8_t mapping_found = 0;
3361
3362         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3363                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3364                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3365                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3366                                         tx_queue_stats_mappings[i].queue_id,
3367                                         tx_queue_stats_mappings[i].stats_counter_id);
3368                         if (diag != 0)
3369                                 return diag;
3370                         mapping_found = 1;
3371                 }
3372         }
3373         if (mapping_found)
3374                 port->tx_queue_stats_mapping_enabled = 1;
3375         return 0;
3376 }
3377
3378 static int
3379 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3380 {
3381         uint16_t i;
3382         int diag;
3383         uint8_t mapping_found = 0;
3384
3385         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3386                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3387                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3388                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3389                                         rx_queue_stats_mappings[i].queue_id,
3390                                         rx_queue_stats_mappings[i].stats_counter_id);
3391                         if (diag != 0)
3392                                 return diag;
3393                         mapping_found = 1;
3394                 }
3395         }
3396         if (mapping_found)
3397                 port->rx_queue_stats_mapping_enabled = 1;
3398         return 0;
3399 }
3400
3401 static void
3402 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3403 {
3404         int diag = 0;
3405
3406         diag = set_tx_queue_stats_mapping_registers(pi, port);
3407         if (diag != 0) {
3408                 if (diag == -ENOTSUP) {
3409                         port->tx_queue_stats_mapping_enabled = 0;
3410                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3411                 }
3412                 else
3413                         rte_exit(EXIT_FAILURE,
3414                                         "set_tx_queue_stats_mapping_registers "
3415                                         "failed for port id=%d diag=%d\n",
3416                                         pi, diag);
3417         }
3418
3419         diag = set_rx_queue_stats_mapping_registers(pi, port);
3420         if (diag != 0) {
3421                 if (diag == -ENOTSUP) {
3422                         port->rx_queue_stats_mapping_enabled = 0;
3423                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3424                 }
3425                 else
3426                         rte_exit(EXIT_FAILURE,
3427                                         "set_rx_queue_stats_mapping_registers "
3428                                         "failed for port id=%d diag=%d\n",
3429                                         pi, diag);
3430         }
3431 }
3432
3433 static void
3434 rxtx_port_config(struct rte_port *port)
3435 {
3436         uint16_t qid;
3437         uint64_t offloads;
3438
3439         for (qid = 0; qid < nb_rxq; qid++) {
3440                 offloads = port->rx_conf[qid].offloads;
3441                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3442                 if (offloads != 0)
3443                         port->rx_conf[qid].offloads = offloads;
3444
3445                 /* Check if any Rx parameters have been passed */
3446                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3447                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3448
3449                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3450                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3451
3452                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3453                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3454
3455                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3456                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3457
3458                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3459                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3460
3461                 port->nb_rx_desc[qid] = nb_rxd;
3462         }
3463
3464         for (qid = 0; qid < nb_txq; qid++) {
3465                 offloads = port->tx_conf[qid].offloads;
3466                 port->tx_conf[qid] = port->dev_info.default_txconf;
3467                 if (offloads != 0)
3468                         port->tx_conf[qid].offloads = offloads;
3469
3470                 /* Check if any Tx parameters have been passed */
3471                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3472                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3473
3474                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3475                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3476
3477                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3478                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3479
3480                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3481                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3482
3483                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3484                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3485
3486                 port->nb_tx_desc[qid] = nb_txd;
3487         }
3488 }
3489
3490 void
3491 init_port_config(void)
3492 {
3493         portid_t pid;
3494         struct rte_port *port;
3495         int ret;
3496
3497         RTE_ETH_FOREACH_DEV(pid) {
3498                 port = &ports[pid];
3499                 port->dev_conf.fdir_conf = fdir_conf;
3500
3501                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3502                 if (ret != 0)
3503                         return;
3504
3505                 if (nb_rxq > 1) {
3506                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3507                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3508                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3509                 } else {
3510                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3511                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3512                 }
3513
3514                 if (port->dcb_flag == 0) {
3515                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3516                                 port->dev_conf.rxmode.mq_mode =
3517                                         (enum rte_eth_rx_mq_mode)
3518                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3519                         else
3520                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3521                 }
3522
3523                 rxtx_port_config(port);
3524
3525                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3526                 if (ret != 0)
3527                         return;
3528
3529                 map_port_queue_stats_mapping_registers(pid, port);
3530 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3531                 rte_pmd_ixgbe_bypass_init(pid);
3532 #endif
3533
3534                 if (lsc_interrupt &&
3535                     (rte_eth_devices[pid].data->dev_flags &
3536                      RTE_ETH_DEV_INTR_LSC))
3537                         port->dev_conf.intr_conf.lsc = 1;
3538                 if (rmv_interrupt &&
3539                     (rte_eth_devices[pid].data->dev_flags &
3540                      RTE_ETH_DEV_INTR_RMV))
3541                         port->dev_conf.intr_conf.rmv = 1;
3542         }
3543 }
3544
3545 void set_port_slave_flag(portid_t slave_pid)
3546 {
3547         struct rte_port *port;
3548
3549         port = &ports[slave_pid];
3550         port->slave_flag = 1;
3551 }
3552
3553 void clear_port_slave_flag(portid_t slave_pid)
3554 {
3555         struct rte_port *port;
3556
3557         port = &ports[slave_pid];
3558         port->slave_flag = 0;
3559 }
3560
3561 uint8_t port_is_bonding_slave(portid_t slave_pid)
3562 {
3563         struct rte_port *port;
3564
3565         port = &ports[slave_pid];
3566         if ((rte_eth_devices[slave_pid].data->dev_flags &
3567             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3568                 return 1;
3569         return 0;
3570 }
3571
3572 const uint16_t vlan_tags[] = {
3573                 0,  1,  2,  3,  4,  5,  6,  7,
3574                 8,  9, 10, 11,  12, 13, 14, 15,
3575                 16, 17, 18, 19, 20, 21, 22, 23,
3576                 24, 25, 26, 27, 28, 29, 30, 31
3577 };
3578
3579 static  int
3580 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3581                  enum dcb_mode_enable dcb_mode,
3582                  enum rte_eth_nb_tcs num_tcs,
3583                  uint8_t pfc_en)
3584 {
3585         uint8_t i;
3586         int32_t rc;
3587         struct rte_eth_rss_conf rss_conf;
3588
3589         /*
3590          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3591          * given above, and the number of traffic classes available for use.
3592          */
3593         if (dcb_mode == DCB_VT_ENABLED) {
3594                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3595                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3596                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3597                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3598
3599                 /* VMDQ+DCB RX and TX configurations */
3600                 vmdq_rx_conf->enable_default_pool = 0;
3601                 vmdq_rx_conf->default_pool = 0;
3602                 vmdq_rx_conf->nb_queue_pools =
3603                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3604                 vmdq_tx_conf->nb_queue_pools =
3605                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3606
3607                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3608                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3609                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3610                         vmdq_rx_conf->pool_map[i].pools =
3611                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3612                 }
3613                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3614                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3615                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3616                 }
3617
3618                 /* set DCB mode of RX and TX of multiple queues */
3619                 eth_conf->rxmode.mq_mode =
3620                                 (enum rte_eth_rx_mq_mode)
3621                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3622                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3623         } else {
3624                 struct rte_eth_dcb_rx_conf *rx_conf =
3625                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3626                 struct rte_eth_dcb_tx_conf *tx_conf =
3627                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3628
3629                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3630
3631                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3632                 if (rc != 0)
3633                         return rc;
3634
3635                 rx_conf->nb_tcs = num_tcs;
3636                 tx_conf->nb_tcs = num_tcs;
3637
3638                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3639                         rx_conf->dcb_tc[i] = i % num_tcs;
3640                         tx_conf->dcb_tc[i] = i % num_tcs;
3641                 }
3642
3643                 eth_conf->rxmode.mq_mode =
3644                                 (enum rte_eth_rx_mq_mode)
3645                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3646                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3647                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3648         }
3649
3650         if (pfc_en)
3651                 eth_conf->dcb_capability_en =
3652                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3653         else
3654                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3655
3656         return 0;
3657 }
3658
3659 int
3660 init_port_dcb_config(portid_t pid,
3661                      enum dcb_mode_enable dcb_mode,
3662                      enum rte_eth_nb_tcs num_tcs,
3663                      uint8_t pfc_en)
3664 {
3665         struct rte_eth_conf port_conf;
3666         struct rte_port *rte_port;
3667         int retval;
3668         uint16_t i;
3669
3670         rte_port = &ports[pid];
3671
3672         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3673         /* Enter DCB configuration status */
3674         dcb_config = 1;
3675
3676         port_conf.rxmode = rte_port->dev_conf.rxmode;
3677         port_conf.txmode = rte_port->dev_conf.txmode;
3678
3679         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3680         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3681         if (retval < 0)
3682                 return retval;
3683         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3684
3685         /* re-configure the device . */
3686         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3687         if (retval < 0)
3688                 return retval;
3689
3690         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3691         if (retval != 0)
3692                 return retval;
3693
3694         /* If dev_info.vmdq_pool_base is greater than 0,
3695          * the queue id of vmdq pools is started after pf queues.
3696          */
3697         if (dcb_mode == DCB_VT_ENABLED &&
3698             rte_port->dev_info.vmdq_pool_base > 0) {
3699                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3700                         " for port %d.", pid);
3701                 return -1;
3702         }
3703
3704         /* Assume the ports in testpmd have the same dcb capability
3705          * and has the same number of rxq and txq in dcb mode
3706          */
3707         if (dcb_mode == DCB_VT_ENABLED) {
3708                 if (rte_port->dev_info.max_vfs > 0) {
3709                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3710                         nb_txq = rte_port->dev_info.nb_tx_queues;
3711                 } else {
3712                         nb_rxq = rte_port->dev_info.max_rx_queues;
3713                         nb_txq = rte_port->dev_info.max_tx_queues;
3714                 }
3715         } else {
3716                 /*if vt is disabled, use all pf queues */
3717                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3718                         nb_rxq = rte_port->dev_info.max_rx_queues;
3719                         nb_txq = rte_port->dev_info.max_tx_queues;
3720                 } else {
3721                         nb_rxq = (queueid_t)num_tcs;
3722                         nb_txq = (queueid_t)num_tcs;
3723
3724                 }
3725         }
3726         rx_free_thresh = 64;
3727
3728         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3729
3730         rxtx_port_config(rte_port);
3731         /* VLAN filter */
3732         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3733         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3734                 rx_vft_set(pid, vlan_tags[i], 1);
3735
3736         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3737         if (retval != 0)
3738                 return retval;
3739
3740         map_port_queue_stats_mapping_registers(pid, rte_port);
3741
3742         rte_port->dcb_flag = 1;
3743
3744         return 0;
3745 }
3746
3747 static void
3748 init_port(void)
3749 {
3750         int i;
3751
3752         /* Configuration of Ethernet ports. */
3753         ports = rte_zmalloc("testpmd: ports",
3754                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3755                             RTE_CACHE_LINE_SIZE);
3756         if (ports == NULL) {
3757                 rte_exit(EXIT_FAILURE,
3758                                 "rte_zmalloc(%d struct rte_port) failed\n",
3759                                 RTE_MAX_ETHPORTS);
3760         }
3761         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3762                 LIST_INIT(&ports[i].flow_tunnel_list);
3763         /* Initialize ports NUMA structures */
3764         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3765         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3766         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3767 }
3768
3769 static void
3770 force_quit(void)
3771 {
3772         pmd_test_exit();
3773         prompt_exit();
3774 }
3775
3776 static void
3777 print_stats(void)
3778 {
3779         uint8_t i;
3780         const char clr[] = { 27, '[', '2', 'J', '\0' };
3781         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3782
3783         /* Clear screen and move to top left */
3784         printf("%s%s", clr, top_left);
3785
3786         printf("\nPort statistics ====================================");
3787         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3788                 nic_stats_display(fwd_ports_ids[i]);
3789
3790         fflush(stdout);
3791 }
3792
3793 static void
3794 signal_handler(int signum)
3795 {
3796         if (signum == SIGINT || signum == SIGTERM) {
3797                 printf("\nSignal %d received, preparing to exit...\n",
3798                                 signum);
3799 #ifdef RTE_LIB_PDUMP
3800                 /* uninitialize packet capture framework */
3801                 rte_pdump_uninit();
3802 #endif
3803 #ifdef RTE_LIB_LATENCYSTATS
3804                 if (latencystats_enabled != 0)
3805                         rte_latencystats_uninit();
3806 #endif
3807                 force_quit();
3808                 /* Set flag to indicate the force termination. */
3809                 f_quit = 1;
3810                 /* exit with the expected status */
3811                 signal(signum, SIG_DFL);
3812                 kill(getpid(), signum);
3813         }
3814 }
3815
3816 int
3817 main(int argc, char** argv)
3818 {
3819         int diag;
3820         portid_t port_id;
3821         uint16_t count;
3822         int ret;
3823
3824         signal(SIGINT, signal_handler);
3825         signal(SIGTERM, signal_handler);
3826
3827         testpmd_logtype = rte_log_register("testpmd");
3828         if (testpmd_logtype < 0)
3829                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3830         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3831
3832         diag = rte_eal_init(argc, argv);
3833         if (diag < 0)
3834                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3835                          rte_strerror(rte_errno));
3836
3837         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3838                 rte_exit(EXIT_FAILURE,
3839                          "Secondary process type not supported.\n");
3840
3841         ret = register_eth_event_callback();
3842         if (ret != 0)
3843                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3844
3845 #ifdef RTE_LIB_PDUMP
3846         /* initialize packet capture framework */
3847         rte_pdump_init();
3848 #endif
3849
3850         count = 0;
3851         RTE_ETH_FOREACH_DEV(port_id) {
3852                 ports_ids[count] = port_id;
3853                 count++;
3854         }
3855         nb_ports = (portid_t) count;
3856         if (nb_ports == 0)
3857                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3858
3859         /* allocate port structures, and init them */
3860         init_port();
3861
3862         set_def_fwd_config();
3863         if (nb_lcores == 0)
3864                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3865                          "Check the core mask argument\n");
3866
3867         /* Bitrate/latency stats disabled by default */
3868 #ifdef RTE_LIB_BITRATESTATS
3869         bitrate_enabled = 0;
3870 #endif
3871 #ifdef RTE_LIB_LATENCYSTATS
3872         latencystats_enabled = 0;
3873 #endif
3874
3875         /* on FreeBSD, mlockall() is disabled by default */
3876 #ifdef RTE_EXEC_ENV_FREEBSD
3877         do_mlockall = 0;
3878 #else
3879         do_mlockall = 1;
3880 #endif
3881
3882         argc -= diag;
3883         argv += diag;
3884         if (argc > 1)
3885                 launch_args_parse(argc, argv);
3886
3887         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3888                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3889                         strerror(errno));
3890         }
3891
3892         if (tx_first && interactive)
3893                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3894                                 "interactive mode.\n");
3895
3896         if (tx_first && lsc_interrupt) {
3897                 printf("Warning: lsc_interrupt needs to be off when "
3898                                 " using tx_first. Disabling.\n");
3899                 lsc_interrupt = 0;
3900         }
3901
3902         if (!nb_rxq && !nb_txq)
3903                 printf("Warning: Either rx or tx queues should be non-zero\n");
3904
3905         if (nb_rxq > 1 && nb_rxq > nb_txq)
3906                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3907                        "but nb_txq=%d will prevent to fully test it.\n",
3908                        nb_rxq, nb_txq);
3909
3910         init_config();
3911
3912         if (hot_plug) {
3913                 ret = rte_dev_hotplug_handle_enable();
3914                 if (ret) {
3915                         RTE_LOG(ERR, EAL,
3916                                 "fail to enable hotplug handling.");
3917                         return -1;
3918                 }
3919
3920                 ret = rte_dev_event_monitor_start();
3921                 if (ret) {
3922                         RTE_LOG(ERR, EAL,
3923                                 "fail to start device event monitoring.");
3924                         return -1;
3925                 }
3926
3927                 ret = rte_dev_event_callback_register(NULL,
3928                         dev_event_callback, NULL);
3929                 if (ret) {
3930                         RTE_LOG(ERR, EAL,
3931                                 "fail  to register device event callback\n");
3932                         return -1;
3933                 }
3934         }
3935
3936         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3937                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3938
3939         /* set all ports to promiscuous mode by default */
3940         RTE_ETH_FOREACH_DEV(port_id) {
3941                 ret = rte_eth_promiscuous_enable(port_id);
3942                 if (ret != 0)
3943                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3944                                 port_id, rte_strerror(-ret));
3945         }
3946
3947         /* Init metrics library */
3948         rte_metrics_init(rte_socket_id());
3949
3950 #ifdef RTE_LIB_LATENCYSTATS
3951         if (latencystats_enabled != 0) {
3952                 int ret = rte_latencystats_init(1, NULL);
3953                 if (ret)
3954                         printf("Warning: latencystats init()"
3955                                 " returned error %d\n", ret);
3956                 printf("Latencystats running on lcore %d\n",
3957                         latencystats_lcore_id);
3958         }
3959 #endif
3960
3961         /* Setup bitrate stats */
3962 #ifdef RTE_LIB_BITRATESTATS
3963         if (bitrate_enabled != 0) {
3964                 bitrate_data = rte_stats_bitrate_create();
3965                 if (bitrate_data == NULL)
3966                         rte_exit(EXIT_FAILURE,
3967                                 "Could not allocate bitrate data.\n");
3968                 rte_stats_bitrate_reg(bitrate_data);
3969         }
3970 #endif
3971
3972 #ifdef RTE_LIB_CMDLINE
3973         if (strlen(cmdline_filename) != 0)
3974                 cmdline_read_from_file(cmdline_filename);
3975
3976         if (interactive == 1) {
3977                 if (auto_start) {
3978                         printf("Start automatic packet forwarding\n");
3979                         start_packet_forwarding(0);
3980                 }
3981                 prompt();
3982                 pmd_test_exit();
3983         } else
3984 #endif
3985         {
3986                 char c;
3987                 int rc;
3988
3989                 f_quit = 0;
3990
3991                 printf("No commandline core given, start packet forwarding\n");
3992                 start_packet_forwarding(tx_first);
3993                 if (stats_period != 0) {
3994                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3995                         uint64_t timer_period;
3996
3997                         /* Convert to number of cycles */
3998                         timer_period = stats_period * rte_get_timer_hz();
3999
4000                         while (f_quit == 0) {
4001                                 cur_time = rte_get_timer_cycles();
4002                                 diff_time += cur_time - prev_time;
4003
4004                                 if (diff_time >= timer_period) {
4005                                         print_stats();
4006                                         /* Reset the timer */
4007                                         diff_time = 0;
4008                                 }
4009                                 /* Sleep to avoid unnecessary checks */
4010                                 prev_time = cur_time;
4011                                 sleep(1);
4012                         }
4013                 }
4014
4015                 printf("Press enter to exit\n");
4016                 rc = read(0, &c, 1);
4017                 pmd_test_exit();
4018                 if (rc < 0)
4019                         return 1;
4020         }
4021
4022         ret = rte_eal_cleanup();
4023         if (ret != 0)
4024                 rte_exit(EXIT_FAILURE,
4025                          "EAL cleanup failed: %s\n", strerror(-ret));
4026
4027         return EXIT_SUCCESS;
4028 }