app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_malloc_heap.h>
  31 #include <rte_memory.h>
  32 #include <rte_memcpy.h>
  33 #include <rte_launch.h>
  34 #include <rte_eal.h>
  35 #include <rte_alarm.h>
  36 #include <rte_per_lcore.h>
  37 #include <rte_lcore.h>
  38 #include <rte_atomic.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_LIBRTE_IXGBE_PMD
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIBRTE_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #include <rte_metrics.h>
  58 #ifdef RTE_LIBRTE_BITRATE
  59 #include <rte_bitrate.h>
  60 #endif
  61 #ifdef RTE_LIBRTE_LATENCY_STATS
  62 #include <rte_latencystats.h>
  63 #endif
  64
  65 #include "testpmd.h"
  66
  67 #ifndef MAP_HUGETLB
  68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  69 #define HUGE_FLAG (0x40000)
  70 #else
  71 #define HUGE_FLAG MAP_HUGETLB
  72 #endif
  73
  74 #ifndef MAP_HUGE_SHIFT
  75 /* older kernels (or FreeBSD) will not have this define */
  76 #define HUGE_SHIFT (26)
  77 #else
  78 #define HUGE_SHIFT MAP_HUGE_SHIFT
  79 #endif
  80
  81 #define EXTMEM_HEAP_NAME "extmem"
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192 uint16_t mempool_flags;
 193
 194 struct fwd_config cur_fwd_config;
 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 196 uint32_t retry_enabled;
 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 199
 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 201 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 202                                       * specified on command-line. */
 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 204
 205 /*
 206  * In container, it cannot terminate the process which running with 'stats-period'
 207  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 208  */
 209 uint8_t f_quit;
 210
 211 /*
 212  * Configuration of packet segments used by the "txonly" processing engine.
 213  */
 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 216         TXONLY_DEF_PACKET_LEN,
 217 };
 218 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 219
 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 221 /**< Split policy for packets to TX. */
 222
 223 uint8_t txonly_multi_flow;
 224 /**< Whether multiple flows are generated in TXONLY mode. */
 225
 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 228
 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 230 uint8_t dcb_config = 0;
 231
 232 /* Whether the dcb is in testing status */
 233 uint8_t dcb_test = 0;
 234
 235 /*
 236  * Configurable number of RX/TX queues.
 237  */
 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 240
 241 /*
 242  * Configurable number of RX/TX ring descriptors.
 243  * Defaults are supplied by drivers via ethdev.
 244  */
 245 #define RTE_TEST_RX_DESC_DEFAULT 0
 246 #define RTE_TEST_TX_DESC_DEFAULT 0
 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 249
 250 #define RTE_PMD_PARAM_UNSET -1
 251 /*
 252  * Configurable values of RX and TX ring threshold registers.
 253  */
 254
 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 258
 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 262
 263 /*
 264  * Configurable value of RX free threshold.
 265  */
 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 267
 268 /*
 269  * Configurable value of RX drop enable.
 270  */
 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 272
 273 /*
 274  * Configurable value of TX free threshold.
 275  */
 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of TX RS bit threshold.
 280  */
 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of buffered packets before sending.
 285  */
 286 uint16_t noisy_tx_sw_bufsz;
 287
 288 /*
 289  * Configurable value of packet buffer timeout.
 290  */
 291 uint16_t noisy_tx_sw_buf_flush_time;
 292
 293 /*
 294  * Configurable value for size of VNF internal memory area
 295  * used for simulating noisy neighbour behaviour
 296  */
 297 uint64_t noisy_lkup_mem_sz;
 298
 299 /*
 300  * Configurable value of number of random writes done in
 301  * VNF simulation memory area.
 302  */
 303 uint64_t noisy_lkup_num_writes;
 304
 305 /*
 306  * Configurable value of number of random reads done in
 307  * VNF simulation memory area.
 308  */
 309 uint64_t noisy_lkup_num_reads;
 310
 311 /*
 312  * Configurable value of number of random reads/writes done in
 313  * VNF simulation memory area.
 314  */
 315 uint64_t noisy_lkup_num_reads_writes;
 316
 317 /*
 318  * Receive Side Scaling (RSS) configuration.
 319  */
 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 321
 322 /*
 323  * Port topology configuration
 324  */
 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 326
 327 /*
 328  * Avoids to flush all the RX streams before starts forwarding.
 329  */
 330 uint8_t no_flush_rx = 0; /* flush by default */
 331
 332 /*
 333  * Flow API isolated mode.
 334  */
 335 uint8_t flow_isolate_all;
 336
 337 /*
 338  * Avoids to check link status when starting/stopping a port.
 339  */
 340 uint8_t no_link_check = 0; /* check by default */
 341
 342 /*
 343  * Enable link status change notification
 344  */
 345 uint8_t lsc_interrupt = 1; /* enabled by default */
 346
 347 /*
 348  * Enable device removal notification.
 349  */
 350 uint8_t rmv_interrupt = 1; /* enabled by default */
 351
 352 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 353
 354 /* After attach, port setup is called on event or by iterator */
 355 bool setup_on_probe_event = true;
 356
 357 /* Pretty printing of ethdev events */
 358 static const char * const eth_event_desc[] = {
 359         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 360         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 361         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 362         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 363         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 364         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 365         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 366         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 367         [RTE_ETH_EVENT_NEW] = "device probed",
 368         [RTE_ETH_EVENT_DESTROY] = "device released",
 369         [RTE_ETH_EVENT_MAX] = NULL,
 370 };
 371
 372 /*
 373  * Display or mask ether events
 374  * Default to all events except VF_MBOX
 375  */
 376 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 377                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 378                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 379                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 380                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 381                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 382                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 383 /*
 384  * Decide if all memory are locked for performance.
 385  */
 386 int do_mlockall = 0;
 387
 388 /*
 389  * NIC bypass mode configuration options.
 390  */
 391
 392 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 393 /* The NIC bypass watchdog timeout. */
 394 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 395 #endif
 396
 397
 398 #ifdef RTE_LIBRTE_LATENCY_STATS
 399
 400 /*
 401  * Set when latency stats is enabled in the commandline
 402  */
 403 uint8_t latencystats_enabled;
 404
 405 /*
 406  * Lcore ID to serive latency statistics.
 407  */
 408 lcoreid_t latencystats_lcore_id = -1;
 409
 410 #endif
 411
 412 /*
 413  * Ethernet device configuration.
 414  */
 415 struct rte_eth_rxmode rx_mode = {
 416         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 417                 /**< Default maximum frame length. */
 418 };
 419
 420 struct rte_eth_txmode tx_mode = {
 421         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 422 };
 423
 424 struct rte_fdir_conf fdir_conf = {
 425         .mode = RTE_FDIR_MODE_NONE,
 426         .pballoc = RTE_FDIR_PBALLOC_64K,
 427         .status = RTE_FDIR_REPORT_STATUS,
 428         .mask = {
 429                 .vlan_tci_mask = 0xFFEF,
 430                 .ipv4_mask     = {
 431                         .src_ip = 0xFFFFFFFF,
 432                         .dst_ip = 0xFFFFFFFF,
 433                 },
 434                 .ipv6_mask     = {
 435                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 436                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 437                 },
 438                 .src_port_mask = 0xFFFF,
 439                 .dst_port_mask = 0xFFFF,
 440                 .mac_addr_byte_mask = 0xFF,
 441                 .tunnel_type_mask = 1,
 442                 .tunnel_id_mask = 0xFFFFFFFF,
 443         },
 444         .drop_queue = 127,
 445 };
 446
 447 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 448
 449 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 450 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 451
 452 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 453 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 454
 455 uint16_t nb_tx_queue_stats_mappings = 0;
 456 uint16_t nb_rx_queue_stats_mappings = 0;
 457
 458 /*
 459  * Display zero values by default for xstats
 460  */
 461 uint8_t xstats_hide_zero;
 462
 463 unsigned int num_sockets = 0;
 464 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 465
 466 #ifdef RTE_LIBRTE_BITRATE
 467 /* Bitrate statistics */
 468 struct rte_stats_bitrates *bitrate_data;
 469 lcoreid_t bitrate_lcore_id;
 470 uint8_t bitrate_enabled;
 471 #endif
 472
 473 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 474 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 475
 476 struct vxlan_encap_conf vxlan_encap_conf = {
 477         .select_ipv4 = 1,
 478         .select_vlan = 0,
 479         .select_tos_ttl = 0,
 480         .vni = "\x00\x00\x00",
 481         .udp_src = 0,
 482         .udp_dst = RTE_BE16(4789),
 483         .ipv4_src = RTE_IPV4(127, 0, 0, 1),
 484         .ipv4_dst = RTE_IPV4(255, 255, 255, 255),
 485         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 486                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 487         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 488                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 489         .vlan_tci = 0,
 490         .ip_tos = 0,
 491         .ip_ttl = 255,
 492         .eth_src = "\x00\x00\x00\x00\x00\x00",
 493         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 494 };
 495
 496 struct nvgre_encap_conf nvgre_encap_conf = {
 497         .select_ipv4 = 1,
 498         .select_vlan = 0,
 499         .tni = "\x00\x00\x00",
 500         .ipv4_src = RTE_IPV4(127, 0, 0, 1),
 501         .ipv4_dst = RTE_IPV4(255, 255, 255, 255),
 502         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 503                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 504         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 505                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 506         .vlan_tci = 0,
 507         .eth_src = "\x00\x00\x00\x00\x00\x00",
 508         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 509 };
 510
 511 /* Forward function declarations */
 512 static void setup_attached_port(portid_t pi);
 513 static void map_port_queue_stats_mapping_registers(portid_t pi,
 514                                                    struct rte_port *port);
 515 static void check_all_ports_link_status(uint32_t port_mask);
 516 static int eth_event_callback(portid_t port_id,
 517                               enum rte_eth_event_type type,
 518                               void *param, void *ret_param);
 519 static void dev_event_callback(const char *device_name,
 520                                 enum rte_dev_event_type type,
 521                                 void *param);
 522
 523 /*
 524  * Check if all the ports are started.
 525  * If yes, return positive value. If not, return zero.
 526  */
 527 static int all_ports_started(void);
 528
 529 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 530 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 531
 532 /*
 533  * Helper function to check if socket is already discovered.
 534  * If yes, return positive value. If not, return zero.
 535  */
 536 int
 537 new_socket_id(unsigned int socket_id)
 538 {
 539         unsigned int i;
 540
 541         for (i = 0; i < num_sockets; i++) {
 542                 if (socket_ids[i] == socket_id)
 543                         return 0;
 544         }
 545         return 1;
 546 }
 547
 548 /*
 549  * Setup default configuration.
 550  */
 551 static void
 552 set_default_fwd_lcores_config(void)
 553 {
 554         unsigned int i;
 555         unsigned int nb_lc;
 556         unsigned int sock_num;
 557
 558         nb_lc = 0;
 559         for (i = 0; i < RTE_MAX_LCORE; i++) {
 560                 if (!rte_lcore_is_enabled(i))
 561                         continue;
 562                 sock_num = rte_lcore_to_socket_id(i);
 563                 if (new_socket_id(sock_num)) {
 564                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 565                                 rte_exit(EXIT_FAILURE,
 566                                          "Total sockets greater than %u\n",
 567                                          RTE_MAX_NUMA_NODES);
 568                         }
 569                         socket_ids[num_sockets++] = sock_num;
 570                 }
 571                 if (i == rte_get_master_lcore())
 572                         continue;
 573                 fwd_lcores_cpuids[nb_lc++] = i;
 574         }
 575         nb_lcores = (lcoreid_t) nb_lc;
 576         nb_cfg_lcores = nb_lcores;
 577         nb_fwd_lcores = 1;
 578 }
 579
 580 static void
 581 set_def_peer_eth_addrs(void)
 582 {
 583         portid_t i;
 584
 585         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 586                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 587                 peer_eth_addrs[i].addr_bytes[5] = i;
 588         }
 589 }
 590
 591 static void
 592 set_default_fwd_ports_config(void)
 593 {
 594         portid_t pt_id;
 595         int i = 0;
 596
 597         RTE_ETH_FOREACH_DEV(pt_id) {
 598                 fwd_ports_ids[i++] = pt_id;
 599
 600                 /* Update sockets info according to the attached device */
 601                 int socket_id = rte_eth_dev_socket_id(pt_id);
 602                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 603                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 604                                 rte_exit(EXIT_FAILURE,
 605                                          "Total sockets greater than %u\n",
 606                                          RTE_MAX_NUMA_NODES);
 607                         }
 608                         socket_ids[num_sockets++] = socket_id;
 609                 }
 610         }
 611
 612         nb_cfg_ports = nb_ports;
 613         nb_fwd_ports = nb_ports;
 614 }
 615
 616 void
 617 set_def_fwd_config(void)
 618 {
 619         set_default_fwd_lcores_config();
 620         set_def_peer_eth_addrs();
 621         set_default_fwd_ports_config();
 622 }
 623
 624 /* extremely pessimistic estimation of memory required to create a mempool */
 625 static int
 626 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 627 {
 628         unsigned int n_pages, mbuf_per_pg, leftover;
 629         uint64_t total_mem, mbuf_mem, obj_sz;
 630
 631         /* there is no good way to predict how much space the mempool will
 632          * occupy because it will allocate chunks on the fly, and some of those
 633          * will come from default DPDK memory while some will come from our
 634          * external memory, so just assume 128MB will be enough for everyone.
 635          */
 636         uint64_t hdr_mem = 128 << 20;
 637
 638         /* account for possible non-contiguousness */
 639         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 640         if (obj_sz > pgsz) {
 641                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 642                 return -1;
 643         }
 644
 645         mbuf_per_pg = pgsz / obj_sz;
 646         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 647         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 648
 649         mbuf_mem = n_pages * pgsz;
 650
 651         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 652
 653         if (total_mem > SIZE_MAX) {
 654                 TESTPMD_LOG(ERR, "Memory size too big\n");
 655                 return -1;
 656         }
 657         *out = (size_t)total_mem;
 658
 659         return 0;
 660 }
 661
 662 static int
 663 pagesz_flags(uint64_t page_sz)
 664 {
 665         /* as per mmap() manpage, all page sizes are log2 of page size
 666          * shifted by MAP_HUGE_SHIFT
 667          */
 668         int log2 = rte_log2_u64(page_sz);
 669
 670         return (log2 << HUGE_SHIFT);
 671 }
 672
 673 static void *
 674 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 675 {
 676         void *addr;
 677         int flags;
 678
 679         /* allocate anonymous hugepages */
 680         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 681         if (huge)
 682                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 683
 684         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 685         if (addr == MAP_FAILED)
 686                 return NULL;
 687
 688         return addr;
 689 }
 690
 691 struct extmem_param {
 692         void *addr;
 693         size_t len;
 694         size_t pgsz;
 695         rte_iova_t *iova_table;
 696         unsigned int iova_table_len;
 697 };
 698
 699 static int
 700 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 701                 bool huge)
 702 {
 703         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 704                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 705         unsigned int cur_page, n_pages, pgsz_idx;
 706         size_t mem_sz, cur_pgsz;
 707         rte_iova_t *iovas = NULL;
 708         void *addr;
 709         int ret;
 710
 711         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 712                 /* skip anything that is too big */
 713                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 714                         continue;
 715
 716                 cur_pgsz = pgsizes[pgsz_idx];
 717
 718                 /* if we were told not to allocate hugepages, override */
 719                 if (!huge)
 720                         cur_pgsz = sysconf(_SC_PAGESIZE);
 721
 722                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 723                 if (ret < 0) {
 724                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 725                         return -1;
 726                 }
 727
 728                 /* allocate our memory */
 729                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 730
 731                 /* if we couldn't allocate memory with a specified page size,
 732                  * that doesn't mean we can't do it with other page sizes, so
 733                  * try another one.
 734                  */
 735                 if (addr == NULL)
 736                         continue;
 737
 738                 /* store IOVA addresses for every page in this memory area */
 739                 n_pages = mem_sz / cur_pgsz;
 740
 741                 iovas = malloc(sizeof(*iovas) * n_pages);
 742
 743                 if (iovas == NULL) {
 744                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 745                         goto fail;
 746                 }
 747                 /* lock memory if it's not huge pages */
 748                 if (!huge)
 749                         mlock(addr, mem_sz);
 750
 751                 /* populate IOVA addresses */
 752                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 753                         rte_iova_t iova;
 754                         size_t offset;
 755                         void *cur;
 756
 757                         offset = cur_pgsz * cur_page;
 758                         cur = RTE_PTR_ADD(addr, offset);
 759
 760                         /* touch the page before getting its IOVA */
 761                         *(volatile char *)cur = 0;
 762
 763                         iova = rte_mem_virt2iova(cur);
 764
 765                         iovas[cur_page] = iova;
 766                 }
 767
 768                 break;
 769         }
 770         /* if we couldn't allocate anything */
 771         if (iovas == NULL)
 772                 return -1;
 773
 774         param->addr = addr;
 775         param->len = mem_sz;
 776         param->pgsz = cur_pgsz;
 777         param->iova_table = iovas;
 778         param->iova_table_len = n_pages;
 779
 780         return 0;
 781 fail:
 782         if (iovas)
 783                 free(iovas);
 784         if (addr)
 785                 munmap(addr, mem_sz);
 786
 787         return -1;
 788 }
 789
 790 static int
 791 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 792 {
 793         struct extmem_param param;
 794         int socket_id, ret;
 795
 796         memset(&param, 0, sizeof(param));
 797
 798         /* check if our heap exists */
 799         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 800         if (socket_id < 0) {
 801                 /* create our heap */
 802                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 803                 if (ret < 0) {
 804                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 805                         return -1;
 806                 }
 807         }
 808
 809         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 810         if (ret < 0) {
 811                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 812                 return -1;
 813         }
 814
 815         /* we now have a valid memory area, so add it to heap */
 816         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 817                         param.addr, param.len, param.iova_table,
 818                         param.iova_table_len, param.pgsz);
 819
 820         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 821
 822         /* not needed any more */
 823         free(param.iova_table);
 824
 825         if (ret < 0) {
 826                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 827                 munmap(param.addr, param.len);
 828                 return -1;
 829         }
 830
 831         /* success */
 832
 833         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 834                         param.len >> 20);
 835
 836         return 0;
 837 }
 838 static void
 839 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 840              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 841 {
 842         uint16_t pid = 0;
 843         int ret;
 844
 845         RTE_ETH_FOREACH_DEV(pid) {
 846                 struct rte_eth_dev *dev =
 847                         &rte_eth_devices[pid];
 848
 849                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 850                                         memhdr->len);
 851                 if (ret) {
 852                         TESTPMD_LOG(DEBUG,
 853                                     "unable to DMA unmap addr 0x%p "
 854                                     "for device %s\n",
 855                                     memhdr->addr, dev->data->name);
 856                 }
 857         }
 858         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 859         if (ret) {
 860                 TESTPMD_LOG(DEBUG,
 861                             "unable to un-register addr 0x%p\n", memhdr->addr);
 862         }
 863 }
 864
 865 static void
 866 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 867            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 868 {
 869         uint16_t pid = 0;
 870         size_t page_size = sysconf(_SC_PAGESIZE);
 871         int ret;
 872
 873         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 874                                   page_size);
 875         if (ret) {
 876                 TESTPMD_LOG(DEBUG,
 877                             "unable to register addr 0x%p\n", memhdr->addr);
 878                 return;
 879         }
 880         RTE_ETH_FOREACH_DEV(pid) {
 881                 struct rte_eth_dev *dev =
 882                         &rte_eth_devices[pid];
 883
 884                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 885                                       memhdr->len);
 886                 if (ret) {
 887                         TESTPMD_LOG(DEBUG,
 888                                     "unable to DMA map addr 0x%p "
 889                                     "for device %s\n",
 890                                     memhdr->addr, dev->data->name);
 891                 }
 892         }
 893 }
 894
 895 /*
 896  * Configuration initialisation done once at init time.
 897  */
 898 static struct rte_mempool *
 899 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 900                  unsigned int socket_id)
 901 {
 902         char pool_name[RTE_MEMPOOL_NAMESIZE];
 903         struct rte_mempool *rte_mp = NULL;
 904         uint32_t mb_size;
 905
 906         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 907         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 908
 909         TESTPMD_LOG(INFO,
 910                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 911                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 912
 913         switch (mp_alloc_type) {
 914         case MP_ALLOC_NATIVE:
 915                 {
 916                         /* wrapper to rte_mempool_create() */
 917                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 918                                         rte_mbuf_best_mempool_ops());
 919                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 920                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 921                         break;
 922                 }
 923         case MP_ALLOC_ANON:
 924                 {
 925                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 926                                 mb_size, (unsigned int) mb_mempool_cache,
 927                                 sizeof(struct rte_pktmbuf_pool_private),
 928                                 socket_id, mempool_flags);
 929                         if (rte_mp == NULL)
 930                                 goto err;
 931
 932                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 933                                 rte_mempool_free(rte_mp);
 934                                 rte_mp = NULL;
 935                                 goto err;
 936                         }
 937                         rte_pktmbuf_pool_init(rte_mp, NULL);
 938                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 939                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 940                         break;
 941                 }
 942         case MP_ALLOC_XMEM:
 943         case MP_ALLOC_XMEM_HUGE:
 944                 {
 945                         int heap_socket;
 946                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 947
 948                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 949                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 950
 951                         heap_socket =
 952                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 953                         if (heap_socket < 0)
 954                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 955
 956                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 957                                         rte_mbuf_best_mempool_ops());
 958                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 959                                         mb_mempool_cache, 0, mbuf_seg_size,
 960                                         heap_socket);
 961                         break;
 962                 }
 963         default:
 964                 {
 965                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 966                 }
 967         }
 968
 969 err:
 970         if (rte_mp == NULL) {
 971                 rte_exit(EXIT_FAILURE,
 972                         "Creation of mbuf pool for socket %u failed: %s\n",
 973                         socket_id, rte_strerror(rte_errno));
 974         } else if (verbose_level > 0) {
 975                 rte_mempool_dump(stdout, rte_mp);
 976         }
 977         return rte_mp;
 978 }
 979
 980 /*
 981  * Check given socket id is valid or not with NUMA mode,
 982  * if valid, return 0, else return -1
 983  */
 984 static int
 985 check_socket_id(const unsigned int socket_id)
 986 {
 987         static int warning_once = 0;
 988
 989         if (new_socket_id(socket_id)) {
 990                 if (!warning_once && numa_support)
 991                         printf("Warning: NUMA should be configured manually by"
 992                                " using --port-numa-config and"
 993                                " --ring-numa-config parameters along with"
 994                                " --numa.\n");
 995                 warning_once = 1;
 996                 return -1;
 997         }
 998         return 0;
 999 }
1000
1001 /*
1002  * Get the allowed maximum number of RX queues.
1003  * *pid return the port id which has minimal value of
1004  * max_rx_queues in all ports.
1005  */
1006 queueid_t
1007 get_allowed_max_nb_rxq(portid_t *pid)
1008 {
1009         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
1010         portid_t pi;
1011         struct rte_eth_dev_info dev_info;
1012
1013         RTE_ETH_FOREACH_DEV(pi) {
1014                 rte_eth_dev_info_get(pi, &dev_info);
1015                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1016                         allowed_max_rxq = dev_info.max_rx_queues;
1017                         *pid = pi;
1018                 }
1019         }
1020         return allowed_max_rxq;
1021 }
1022
1023 /*
1024  * Check input rxq is valid or not.
1025  * If input rxq is not greater than any of maximum number
1026  * of RX queues of all ports, it is valid.
1027  * if valid, return 0, else return -1
1028  */
1029 int
1030 check_nb_rxq(queueid_t rxq)
1031 {
1032         queueid_t allowed_max_rxq;
1033         portid_t pid = 0;
1034
1035         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1036         if (rxq > allowed_max_rxq) {
1037                 printf("Fail: input rxq (%u) can't be greater "
1038                        "than max_rx_queues (%u) of port %u\n",
1039                        rxq,
1040                        allowed_max_rxq,
1041                        pid);
1042                 return -1;
1043         }
1044         return 0;
1045 }
1046
1047 /*
1048  * Get the allowed maximum number of TX queues.
1049  * *pid return the port id which has minimal value of
1050  * max_tx_queues in all ports.
1051  */
1052 queueid_t
1053 get_allowed_max_nb_txq(portid_t *pid)
1054 {
1055         queueid_t allowed_max_txq = MAX_QUEUE_ID;
1056         portid_t pi;
1057         struct rte_eth_dev_info dev_info;
1058
1059         RTE_ETH_FOREACH_DEV(pi) {
1060                 rte_eth_dev_info_get(pi, &dev_info);
1061                 if (dev_info.max_tx_queues < allowed_max_txq) {
1062                         allowed_max_txq = dev_info.max_tx_queues;
1063                         *pid = pi;
1064                 }
1065         }
1066         return allowed_max_txq;
1067 }
1068
1069 /*
1070  * Check input txq is valid or not.
1071  * If input txq is not greater than any of maximum number
1072  * of TX queues of all ports, it is valid.
1073  * if valid, return 0, else return -1
1074  */
1075 int
1076 check_nb_txq(queueid_t txq)
1077 {
1078         queueid_t allowed_max_txq;
1079         portid_t pid = 0;
1080
1081         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1082         if (txq > allowed_max_txq) {
1083                 printf("Fail: input txq (%u) can't be greater "
1084                        "than max_tx_queues (%u) of port %u\n",
1085                        txq,
1086                        allowed_max_txq,
1087                        pid);
1088                 return -1;
1089         }
1090         return 0;
1091 }
1092
1093 static void
1094 init_config(void)
1095 {
1096         portid_t pid;
1097         struct rte_port *port;
1098         struct rte_mempool *mbp;
1099         unsigned int nb_mbuf_per_pool;
1100         lcoreid_t  lc_id;
1101         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1102         struct rte_gro_param gro_param;
1103         uint32_t gso_types;
1104         uint16_t data_size;
1105         bool warning = 0;
1106         int k;
1107
1108         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1109
1110         /* Configuration of logical cores. */
1111         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1112                                 sizeof(struct fwd_lcore *) * nb_lcores,
1113                                 RTE_CACHE_LINE_SIZE);
1114         if (fwd_lcores == NULL) {
1115                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1116                                                         "failed\n", nb_lcores);
1117         }
1118         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1119                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1120                                                sizeof(struct fwd_lcore),
1121                                                RTE_CACHE_LINE_SIZE);
1122                 if (fwd_lcores[lc_id] == NULL) {
1123                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1124                                                                 "failed\n");
1125                 }
1126                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1127         }
1128
1129         RTE_ETH_FOREACH_DEV(pid) {
1130                 port = &ports[pid];
1131                 /* Apply default TxRx configuration for all ports */
1132                 port->dev_conf.txmode = tx_mode;
1133                 port->dev_conf.rxmode = rx_mode;
1134                 rte_eth_dev_info_get(pid, &port->dev_info);
1135
1136                 if (!(port->dev_info.tx_offload_capa &
1137                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1138                         port->dev_conf.txmode.offloads &=
1139                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1140                 if (!(port->dev_info.tx_offload_capa &
1141                         DEV_TX_OFFLOAD_MATCH_METADATA))
1142                         port->dev_conf.txmode.offloads &=
1143                                 ~DEV_TX_OFFLOAD_MATCH_METADATA;
1144                 if (numa_support) {
1145                         if (port_numa[pid] != NUMA_NO_CONFIG)
1146                                 port_per_socket[port_numa[pid]]++;
1147                         else {
1148                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1149
1150                                 /*
1151                                  * if socket_id is invalid,
1152                                  * set to the first available socket.
1153                                  */
1154                                 if (check_socket_id(socket_id) < 0)
1155                                         socket_id = socket_ids[0];
1156                                 port_per_socket[socket_id]++;
1157                         }
1158                 }
1159
1160                 /* Apply Rx offloads configuration */
1161                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1162                         port->rx_conf[k].offloads =
1163                                 port->dev_conf.rxmode.offloads;
1164                 /* Apply Tx offloads configuration */
1165                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1166                         port->tx_conf[k].offloads =
1167                                 port->dev_conf.txmode.offloads;
1168
1169                 /* set flag to initialize port/queue */
1170                 port->need_reconfig = 1;
1171                 port->need_reconfig_queues = 1;
1172                 port->tx_metadata = 0;
1173
1174                 /* Check for maximum number of segments per MTU. Accordingly
1175                  * update the mbuf data size.
1176                  */
1177                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1178                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1179                         data_size = rx_mode.max_rx_pkt_len /
1180                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1181
1182                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1183                                                         mbuf_data_size) {
1184                                 mbuf_data_size = data_size +
1185                                                  RTE_PKTMBUF_HEADROOM;
1186                                 warning = 1;
1187                         }
1188                 }
1189         }
1190
1191         if (warning)
1192                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1193                             mbuf_data_size);
1194
1195         /*
1196          * Create pools of mbuf.
1197          * If NUMA support is disabled, create a single pool of mbuf in
1198          * socket 0 memory by default.
1199          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1200          *
1201          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1202          * nb_txd can be configured at run time.
1203          */
1204         if (param_total_num_mbufs)
1205                 nb_mbuf_per_pool = param_total_num_mbufs;
1206         else {
1207                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1208                         (nb_lcores * mb_mempool_cache) +
1209                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1210                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1211         }
1212
1213         if (numa_support) {
1214                 uint8_t i;
1215
1216                 for (i = 0; i < num_sockets; i++)
1217                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1218                                                        nb_mbuf_per_pool,
1219                                                        socket_ids[i]);
1220         } else {
1221                 if (socket_num == UMA_NO_CONFIG)
1222                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1223                                                        nb_mbuf_per_pool, 0);
1224                 else
1225                         mempools[socket_num] = mbuf_pool_create
1226                                                         (mbuf_data_size,
1227                                                          nb_mbuf_per_pool,
1228                                                          socket_num);
1229         }
1230
1231         init_port_config();
1232
1233         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1234                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1235         /*
1236          * Records which Mbuf pool to use by each logical core, if needed.
1237          */
1238         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1239                 mbp = mbuf_pool_find(
1240                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1241
1242                 if (mbp == NULL)
1243                         mbp = mbuf_pool_find(0);
1244                 fwd_lcores[lc_id]->mbp = mbp;
1245                 /* initialize GSO context */
1246                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1247                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1248                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1249                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1250                         RTE_ETHER_CRC_LEN;
1251                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1252         }
1253
1254         /* Configuration of packet forwarding streams. */
1255         if (init_fwd_streams() < 0)
1256                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1257
1258         fwd_config_setup();
1259
1260         /* create a gro context for each lcore */
1261         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1262         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1263         gro_param.max_item_per_flow = MAX_PKT_BURST;
1264         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1265                 gro_param.socket_id = rte_lcore_to_socket_id(
1266                                 fwd_lcores_cpuids[lc_id]);
1267                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1268                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1269                         rte_exit(EXIT_FAILURE,
1270                                         "rte_gro_ctx_create() failed\n");
1271                 }
1272         }
1273
1274 #if defined RTE_LIBRTE_PMD_SOFTNIC
1275         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1276                 RTE_ETH_FOREACH_DEV(pid) {
1277                         port = &ports[pid];
1278                         const char *driver = port->dev_info.driver_name;
1279
1280                         if (strcmp(driver, "net_softnic") == 0)
1281                                 port->softport.fwd_lcore_arg = fwd_lcores;
1282                 }
1283         }
1284 #endif
1285
1286 }
1287
1288
1289 void
1290 reconfig(portid_t new_port_id, unsigned socket_id)
1291 {
1292         struct rte_port *port;
1293
1294         /* Reconfiguration of Ethernet ports. */
1295         port = &ports[new_port_id];
1296         rte_eth_dev_info_get(new_port_id, &port->dev_info);
1297
1298         /* set flag to initialize port/queue */
1299         port->need_reconfig = 1;
1300         port->need_reconfig_queues = 1;
1301         port->socket_id = socket_id;
1302
1303         init_port_config();
1304 }
1305
1306
1307 int
1308 init_fwd_streams(void)
1309 {
1310         portid_t pid;
1311         struct rte_port *port;
1312         streamid_t sm_id, nb_fwd_streams_new;
1313         queueid_t q;
1314
1315         /* set socket id according to numa or not */
1316         RTE_ETH_FOREACH_DEV(pid) {
1317                 port = &ports[pid];
1318                 if (nb_rxq > port->dev_info.max_rx_queues) {
1319                         printf("Fail: nb_rxq(%d) is greater than "
1320                                 "max_rx_queues(%d)\n", nb_rxq,
1321                                 port->dev_info.max_rx_queues);
1322                         return -1;
1323                 }
1324                 if (nb_txq > port->dev_info.max_tx_queues) {
1325                         printf("Fail: nb_txq(%d) is greater than "
1326                                 "max_tx_queues(%d)\n", nb_txq,
1327                                 port->dev_info.max_tx_queues);
1328                         return -1;
1329                 }
1330                 if (numa_support) {
1331                         if (port_numa[pid] != NUMA_NO_CONFIG)
1332                                 port->socket_id = port_numa[pid];
1333                         else {
1334                                 port->socket_id = rte_eth_dev_socket_id(pid);
1335
1336                                 /*
1337                                  * if socket_id is invalid,
1338                                  * set to the first available socket.
1339                                  */
1340                                 if (check_socket_id(port->socket_id) < 0)
1341                                         port->socket_id = socket_ids[0];
1342                         }
1343                 }
1344                 else {
1345                         if (socket_num == UMA_NO_CONFIG)
1346                                 port->socket_id = 0;
1347                         else
1348                                 port->socket_id = socket_num;
1349                 }
1350         }
1351
1352         q = RTE_MAX(nb_rxq, nb_txq);
1353         if (q == 0) {
1354                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1355                 return -1;
1356         }
1357         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1358         if (nb_fwd_streams_new == nb_fwd_streams)
1359                 return 0;
1360         /* clear the old */
1361         if (fwd_streams != NULL) {
1362                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1363                         if (fwd_streams[sm_id] == NULL)
1364                                 continue;
1365                         rte_free(fwd_streams[sm_id]);
1366                         fwd_streams[sm_id] = NULL;
1367                 }
1368                 rte_free(fwd_streams);
1369                 fwd_streams = NULL;
1370         }
1371
1372         /* init new */
1373         nb_fwd_streams = nb_fwd_streams_new;
1374         if (nb_fwd_streams) {
1375                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1376                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1377                         RTE_CACHE_LINE_SIZE);
1378                 if (fwd_streams == NULL)
1379                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1380                                  " (struct fwd_stream *)) failed\n",
1381                                  nb_fwd_streams);
1382
1383                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1384                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1385                                 " struct fwd_stream", sizeof(struct fwd_stream),
1386                                 RTE_CACHE_LINE_SIZE);
1387                         if (fwd_streams[sm_id] == NULL)
1388                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1389                                          "(struct fwd_stream) failed\n");
1390                 }
1391         }
1392
1393         return 0;
1394 }
1395
1396 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1397 static void
1398 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1399 {
1400         unsigned int total_burst;
1401         unsigned int nb_burst;
1402         unsigned int burst_stats[3];
1403         uint16_t pktnb_stats[3];
1404         uint16_t nb_pkt;
1405         int burst_percent[3];
1406
1407         /*
1408          * First compute the total number of packet bursts and the
1409          * two highest numbers of bursts of the same number of packets.
1410          */
1411         total_burst = 0;
1412         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1413         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1414         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1415                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1416                 if (nb_burst == 0)
1417                         continue;
1418                 total_burst += nb_burst;
1419                 if (nb_burst > burst_stats[0]) {
1420                         burst_stats[1] = burst_stats[0];
1421                         pktnb_stats[1] = pktnb_stats[0];
1422                         burst_stats[0] = nb_burst;
1423                         pktnb_stats[0] = nb_pkt;
1424                 } else if (nb_burst > burst_stats[1]) {
1425                         burst_stats[1] = nb_burst;
1426                         pktnb_stats[1] = nb_pkt;
1427                 }
1428         }
1429         if (total_burst == 0)
1430                 return;
1431         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1432         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1433                burst_percent[0], (int) pktnb_stats[0]);
1434         if (burst_stats[0] == total_burst) {
1435                 printf("]\n");
1436                 return;
1437         }
1438         if (burst_stats[0] + burst_stats[1] == total_burst) {
1439                 printf(" + %d%% of %d pkts]\n",
1440                        100 - burst_percent[0], pktnb_stats[1]);
1441                 return;
1442         }
1443         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1444         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1445         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1446                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1447                 return;
1448         }
1449         printf(" + %d%% of %d pkts + %d%% of others]\n",
1450                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1451 }
1452 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1453
1454 static void
1455 fwd_stream_stats_display(streamid_t stream_id)
1456 {
1457         struct fwd_stream *fs;
1458         static const char *fwd_top_stats_border = "-------";
1459
1460         fs = fwd_streams[stream_id];
1461         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1462             (fs->fwd_dropped == 0))
1463                 return;
1464         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1465                "TX Port=%2d/Queue=%2d %s\n",
1466                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1467                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1468         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1469                " TX-dropped: %-14"PRIu64,
1470                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1471
1472         /* if checksum mode */
1473         if (cur_fwd_eng == &csum_fwd_engine) {
1474                 printf("  RX- bad IP checksum: %-14"PRIu64
1475                        "  Rx- bad L4 checksum: %-14"PRIu64
1476                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1477                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1478                         fs->rx_bad_outer_l4_csum);
1479         } else {
1480                 printf("\n");
1481         }
1482
1483 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1484         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1485         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1486 #endif
1487 }
1488
1489 void
1490 fwd_stats_display(void)
1491 {
1492         static const char *fwd_stats_border = "----------------------";
1493         static const char *acc_stats_border = "+++++++++++++++";
1494         struct {
1495                 struct fwd_stream *rx_stream;
1496                 struct fwd_stream *tx_stream;
1497                 uint64_t tx_dropped;
1498                 uint64_t rx_bad_ip_csum;
1499                 uint64_t rx_bad_l4_csum;
1500                 uint64_t rx_bad_outer_l4_csum;
1501         } ports_stats[RTE_MAX_ETHPORTS];
1502         uint64_t total_rx_dropped = 0;
1503         uint64_t total_tx_dropped = 0;
1504         uint64_t total_rx_nombuf = 0;
1505         struct rte_eth_stats stats;
1506 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1507         uint64_t fwd_cycles = 0;
1508 #endif
1509         uint64_t total_recv = 0;
1510         uint64_t total_xmit = 0;
1511         struct rte_port *port;
1512         streamid_t sm_id;
1513         portid_t pt_id;
1514         int i;
1515
1516         memset(ports_stats, 0, sizeof(ports_stats));
1517
1518         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1519                 struct fwd_stream *fs = fwd_streams[sm_id];
1520
1521                 if (cur_fwd_config.nb_fwd_streams >
1522                     cur_fwd_config.nb_fwd_ports) {
1523                         fwd_stream_stats_display(sm_id);
1524                 } else {
1525                         ports_stats[fs->tx_port].tx_stream = fs;
1526                         ports_stats[fs->rx_port].rx_stream = fs;
1527                 }
1528
1529                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1530
1531                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1532                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1533                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1534                                 fs->rx_bad_outer_l4_csum;
1535
1536 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1537                 fwd_cycles += fs->core_cycles;
1538 #endif
1539         }
1540         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1541                 uint8_t j;
1542
1543                 pt_id = fwd_ports_ids[i];
1544                 port = &ports[pt_id];
1545
1546                 rte_eth_stats_get(pt_id, &stats);
1547                 stats.ipackets -= port->stats.ipackets;
1548                 stats.opackets -= port->stats.opackets;
1549                 stats.ibytes -= port->stats.ibytes;
1550                 stats.obytes -= port->stats.obytes;
1551                 stats.imissed -= port->stats.imissed;
1552                 stats.oerrors -= port->stats.oerrors;
1553                 stats.rx_nombuf -= port->stats.rx_nombuf;
1554
1555                 total_recv += stats.ipackets;
1556                 total_xmit += stats.opackets;
1557                 total_rx_dropped += stats.imissed;
1558                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1559                 total_tx_dropped += stats.oerrors;
1560                 total_rx_nombuf  += stats.rx_nombuf;
1561
1562                 printf("\n  %s Forward statistics for port %-2d %s\n",
1563                        fwd_stats_border, pt_id, fwd_stats_border);
1564
1565                 if (!port->rx_queue_stats_mapping_enabled &&
1566                     !port->tx_queue_stats_mapping_enabled) {
1567                         printf("  RX-packets: %-14"PRIu64
1568                                " RX-dropped: %-14"PRIu64
1569                                "RX-total: %-"PRIu64"\n",
1570                                stats.ipackets, stats.imissed,
1571                                stats.ipackets + stats.imissed);
1572
1573                         if (cur_fwd_eng == &csum_fwd_engine)
1574                                 printf("  Bad-ipcsum: %-14"PRIu64
1575                                        " Bad-l4csum: %-14"PRIu64
1576                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1577                                        ports_stats[pt_id].rx_bad_ip_csum,
1578                                        ports_stats[pt_id].rx_bad_l4_csum,
1579                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1580                         if (stats.ierrors + stats.rx_nombuf > 0) {
1581                                 printf("  RX-error: %-"PRIu64"\n",
1582                                        stats.ierrors);
1583                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1584                                        stats.rx_nombuf);
1585                         }
1586
1587                         printf("  TX-packets: %-14"PRIu64
1588                                " TX-dropped: %-14"PRIu64
1589                                "TX-total: %-"PRIu64"\n",
1590                                stats.opackets, ports_stats[pt_id].tx_dropped,
1591                                stats.opackets + ports_stats[pt_id].tx_dropped);
1592                 } else {
1593                         printf("  RX-packets:             %14"PRIu64
1594                                "    RX-dropped:%14"PRIu64
1595                                "    RX-total:%14"PRIu64"\n",
1596                                stats.ipackets, stats.imissed,
1597                                stats.ipackets + stats.imissed);
1598
1599                         if (cur_fwd_eng == &csum_fwd_engine)
1600                                 printf("  Bad-ipcsum:%14"PRIu64
1601                                        "    Bad-l4csum:%14"PRIu64
1602                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1603                                        ports_stats[pt_id].rx_bad_ip_csum,
1604                                        ports_stats[pt_id].rx_bad_l4_csum,
1605                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1606                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1607                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1608                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1609                                        stats.rx_nombuf);
1610                         }
1611
1612                         printf("  TX-packets:             %14"PRIu64
1613                                "    TX-dropped:%14"PRIu64
1614                                "    TX-total:%14"PRIu64"\n",
1615                                stats.opackets, ports_stats[pt_id].tx_dropped,
1616                                stats.opackets + ports_stats[pt_id].tx_dropped);
1617                 }
1618
1619 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1620                 if (ports_stats[pt_id].rx_stream)
1621                         pkt_burst_stats_display("RX",
1622                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1623                 if (ports_stats[pt_id].tx_stream)
1624                         pkt_burst_stats_display("TX",
1625                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1626 #endif
1627
1628                 if (port->rx_queue_stats_mapping_enabled) {
1629                         printf("\n");
1630                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1631                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1632                                        "     RX-errors:%14"PRIu64
1633                                        "    RX-bytes:%14"PRIu64"\n",
1634                                        j, stats.q_ipackets[j],
1635                                        stats.q_errors[j], stats.q_ibytes[j]);
1636                         }
1637                         printf("\n");
1638                 }
1639                 if (port->tx_queue_stats_mapping_enabled) {
1640                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1641                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1642                                        "                                 TX-bytes:%14"
1643                                        PRIu64"\n",
1644                                        j, stats.q_opackets[j],
1645                                        stats.q_obytes[j]);
1646                         }
1647                 }
1648
1649                 printf("  %s--------------------------------%s\n",
1650                        fwd_stats_border, fwd_stats_border);
1651         }
1652
1653         printf("\n  %s Accumulated forward statistics for all ports"
1654                "%s\n",
1655                acc_stats_border, acc_stats_border);
1656         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1657                "%-"PRIu64"\n"
1658                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1659                "%-"PRIu64"\n",
1660                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1661                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1662         if (total_rx_nombuf > 0)
1663                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1664         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1665                "%s\n",
1666                acc_stats_border, acc_stats_border);
1667 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1668         if (total_recv > 0)
1669                 printf("\n  CPU cycles/packet=%u (total cycles="
1670                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1671                        (unsigned int)(fwd_cycles / total_recv),
1672                        fwd_cycles, total_recv);
1673 #endif
1674 }
1675
1676 void
1677 fwd_stats_reset(void)
1678 {
1679         streamid_t sm_id;
1680         portid_t pt_id;
1681         int i;
1682
1683         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1684                 pt_id = fwd_ports_ids[i];
1685                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1686         }
1687         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1688                 struct fwd_stream *fs = fwd_streams[sm_id];
1689
1690                 fs->rx_packets = 0;
1691                 fs->tx_packets = 0;
1692                 fs->fwd_dropped = 0;
1693                 fs->rx_bad_ip_csum = 0;
1694                 fs->rx_bad_l4_csum = 0;
1695                 fs->rx_bad_outer_l4_csum = 0;
1696
1697 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1698                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1699                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1700 #endif
1701 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1702                 fs->core_cycles = 0;
1703 #endif
1704         }
1705 }
1706
1707 static void
1708 flush_fwd_rx_queues(void)
1709 {
1710         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1711         portid_t  rxp;
1712         portid_t port_id;
1713         queueid_t rxq;
1714         uint16_t  nb_rx;
1715         uint16_t  i;
1716         uint8_t   j;
1717         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1718         uint64_t timer_period;
1719
1720         /* convert to number of cycles */
1721         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1722
1723         for (j = 0; j < 2; j++) {
1724                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1725                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1726                                 port_id = fwd_ports_ids[rxp];
1727                                 /**
1728                                 * testpmd can stuck in the below do while loop
1729                                 * if rte_eth_rx_burst() always returns nonzero
1730                                 * packets. So timer is added to exit this loop
1731                                 * after 1sec timer expiry.
1732                                 */
1733                                 prev_tsc = rte_rdtsc();
1734                                 do {
1735                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1736                                                 pkts_burst, MAX_PKT_BURST);
1737                                         for (i = 0; i < nb_rx; i++)
1738                                                 rte_pktmbuf_free(pkts_burst[i]);
1739
1740                                         cur_tsc = rte_rdtsc();
1741                                         diff_tsc = cur_tsc - prev_tsc;
1742                                         timer_tsc += diff_tsc;
1743                                 } while ((nb_rx > 0) &&
1744                                         (timer_tsc < timer_period));
1745                                 timer_tsc = 0;
1746                         }
1747                 }
1748                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1749         }
1750 }
1751
1752 static void
1753 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1754 {
1755         struct fwd_stream **fsm;
1756         streamid_t nb_fs;
1757         streamid_t sm_id;
1758 #ifdef RTE_LIBRTE_BITRATE
1759         uint64_t tics_per_1sec;
1760         uint64_t tics_datum;
1761         uint64_t tics_current;
1762         uint16_t i, cnt_ports;
1763
1764         cnt_ports = nb_ports;
1765         tics_datum = rte_rdtsc();
1766         tics_per_1sec = rte_get_timer_hz();
1767 #endif
1768         fsm = &fwd_streams[fc->stream_idx];
1769         nb_fs = fc->stream_nb;
1770         do {
1771                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1772                         (*pkt_fwd)(fsm[sm_id]);
1773 #ifdef RTE_LIBRTE_BITRATE
1774                 if (bitrate_enabled != 0 &&
1775                                 bitrate_lcore_id == rte_lcore_id()) {
1776                         tics_current = rte_rdtsc();
1777                         if (tics_current - tics_datum >= tics_per_1sec) {
1778                                 /* Periodic bitrate calculation */
1779                                 for (i = 0; i < cnt_ports; i++)
1780                                         rte_stats_bitrate_calc(bitrate_data,
1781                                                 ports_ids[i]);
1782                                 tics_datum = tics_current;
1783                         }
1784                 }
1785 #endif
1786 #ifdef RTE_LIBRTE_LATENCY_STATS
1787                 if (latencystats_enabled != 0 &&
1788                                 latencystats_lcore_id == rte_lcore_id())
1789                         rte_latencystats_update();
1790 #endif
1791
1792         } while (! fc->stopped);
1793 }
1794
1795 static int
1796 start_pkt_forward_on_core(void *fwd_arg)
1797 {
1798         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1799                              cur_fwd_config.fwd_eng->packet_fwd);
1800         return 0;
1801 }
1802
1803 /*
1804  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1805  * Used to start communication flows in network loopback test configurations.
1806  */
1807 static int
1808 run_one_txonly_burst_on_core(void *fwd_arg)
1809 {
1810         struct fwd_lcore *fwd_lc;
1811         struct fwd_lcore tmp_lcore;
1812
1813         fwd_lc = (struct fwd_lcore *) fwd_arg;
1814         tmp_lcore = *fwd_lc;
1815         tmp_lcore.stopped = 1;
1816         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1817         return 0;
1818 }
1819
1820 /*
1821  * Launch packet forwarding:
1822  *     - Setup per-port forwarding context.
1823  *     - launch logical cores with their forwarding configuration.
1824  */
1825 static void
1826 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1827 {
1828         port_fwd_begin_t port_fwd_begin;
1829         unsigned int i;
1830         unsigned int lc_id;
1831         int diag;
1832
1833         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1834         if (port_fwd_begin != NULL) {
1835                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1836                         (*port_fwd_begin)(fwd_ports_ids[i]);
1837         }
1838         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1839                 lc_id = fwd_lcores_cpuids[i];
1840                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1841                         fwd_lcores[i]->stopped = 0;
1842                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1843                                                      fwd_lcores[i], lc_id);
1844                         if (diag != 0)
1845                                 printf("launch lcore %u failed - diag=%d\n",
1846                                        lc_id, diag);
1847                 }
1848         }
1849 }
1850
1851 /*
1852  * Launch packet forwarding configuration.
1853  */
1854 void
1855 start_packet_forwarding(int with_tx_first)
1856 {
1857         port_fwd_begin_t port_fwd_begin;
1858         port_fwd_end_t  port_fwd_end;
1859         struct rte_port *port;
1860         unsigned int i;
1861         portid_t   pt_id;
1862
1863         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1864                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1865
1866         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1867                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1868
1869         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1870                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1871                 (!nb_rxq || !nb_txq))
1872                 rte_exit(EXIT_FAILURE,
1873                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1874                         cur_fwd_eng->fwd_mode_name);
1875
1876         if (all_ports_started() == 0) {
1877                 printf("Not all ports were started\n");
1878                 return;
1879         }
1880         if (test_done == 0) {
1881                 printf("Packet forwarding already started\n");
1882                 return;
1883         }
1884
1885
1886         if(dcb_test) {
1887                 for (i = 0; i < nb_fwd_ports; i++) {
1888                         pt_id = fwd_ports_ids[i];
1889                         port = &ports[pt_id];
1890                         if (!port->dcb_flag) {
1891                                 printf("In DCB mode, all forwarding ports must "
1892                                        "be configured in this mode.\n");
1893                                 return;
1894                         }
1895                 }
1896                 if (nb_fwd_lcores == 1) {
1897                         printf("In DCB mode,the nb forwarding cores "
1898                                "should be larger than 1.\n");
1899                         return;
1900                 }
1901         }
1902         test_done = 0;
1903
1904         fwd_config_setup();
1905
1906         if(!no_flush_rx)
1907                 flush_fwd_rx_queues();
1908
1909         pkt_fwd_config_display(&cur_fwd_config);
1910         rxtx_config_display();
1911
1912         fwd_stats_reset();
1913         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1914                 pt_id = fwd_ports_ids[i];
1915                 port = &ports[pt_id];
1916                 map_port_queue_stats_mapping_registers(pt_id, port);
1917         }
1918         if (with_tx_first) {
1919                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1920                 if (port_fwd_begin != NULL) {
1921                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1922                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1923                 }
1924                 while (with_tx_first--) {
1925                         launch_packet_forwarding(
1926                                         run_one_txonly_burst_on_core);
1927                         rte_eal_mp_wait_lcore();
1928                 }
1929                 port_fwd_end = tx_only_engine.port_fwd_end;
1930                 if (port_fwd_end != NULL) {
1931                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1932                                 (*port_fwd_end)(fwd_ports_ids[i]);
1933                 }
1934         }
1935         launch_packet_forwarding(start_pkt_forward_on_core);
1936 }
1937
1938 void
1939 stop_packet_forwarding(void)
1940 {
1941         port_fwd_end_t port_fwd_end;
1942         lcoreid_t lc_id;
1943         portid_t pt_id;
1944         int i;
1945
1946         if (test_done) {
1947                 printf("Packet forwarding not started\n");
1948                 return;
1949         }
1950         printf("Telling cores to stop...");
1951         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1952                 fwd_lcores[lc_id]->stopped = 1;
1953         printf("\nWaiting for lcores to finish...\n");
1954         rte_eal_mp_wait_lcore();
1955         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1956         if (port_fwd_end != NULL) {
1957                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1958                         pt_id = fwd_ports_ids[i];
1959                         (*port_fwd_end)(pt_id);
1960                 }
1961         }
1962
1963         fwd_stats_display();
1964
1965         printf("\nDone.\n");
1966         test_done = 1;
1967 }
1968
1969 void
1970 dev_set_link_up(portid_t pid)
1971 {
1972         if (rte_eth_dev_set_link_up(pid) < 0)
1973                 printf("\nSet link up fail.\n");
1974 }
1975
1976 void
1977 dev_set_link_down(portid_t pid)
1978 {
1979         if (rte_eth_dev_set_link_down(pid) < 0)
1980                 printf("\nSet link down fail.\n");
1981 }
1982
1983 static int
1984 all_ports_started(void)
1985 {
1986         portid_t pi;
1987         struct rte_port *port;
1988
1989         RTE_ETH_FOREACH_DEV(pi) {
1990                 port = &ports[pi];
1991                 /* Check if there is a port which is not started */
1992                 if ((port->port_status != RTE_PORT_STARTED) &&
1993                         (port->slave_flag == 0))
1994                         return 0;
1995         }
1996
1997         /* No port is not started */
1998         return 1;
1999 }
2000
2001 int
2002 port_is_stopped(portid_t port_id)
2003 {
2004         struct rte_port *port = &ports[port_id];
2005
2006         if ((port->port_status != RTE_PORT_STOPPED) &&
2007             (port->slave_flag == 0))
2008                 return 0;
2009         return 1;
2010 }
2011
2012 int
2013 all_ports_stopped(void)
2014 {
2015         portid_t pi;
2016
2017         RTE_ETH_FOREACH_DEV(pi) {
2018                 if (!port_is_stopped(pi))
2019                         return 0;
2020         }
2021
2022         return 1;
2023 }
2024
2025 int
2026 port_is_started(portid_t port_id)
2027 {
2028         if (port_id_is_invalid(port_id, ENABLED_WARN))
2029                 return 0;
2030
2031         if (ports[port_id].port_status != RTE_PORT_STARTED)
2032                 return 0;
2033
2034         return 1;
2035 }
2036
2037 int
2038 start_port(portid_t pid)
2039 {
2040         int diag, need_check_link_status = -1;
2041         portid_t pi;
2042         queueid_t qi;
2043         struct rte_port *port;
2044         struct rte_ether_addr mac_addr;
2045
2046         if (port_id_is_invalid(pid, ENABLED_WARN))
2047                 return 0;
2048
2049         if(dcb_config)
2050                 dcb_test = 1;
2051         RTE_ETH_FOREACH_DEV(pi) {
2052                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2053                         continue;
2054
2055                 need_check_link_status = 0;
2056                 port = &ports[pi];
2057                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2058                                                  RTE_PORT_HANDLING) == 0) {
2059                         printf("Port %d is now not stopped\n", pi);
2060                         continue;
2061                 }
2062
2063                 if (port->need_reconfig > 0) {
2064                         port->need_reconfig = 0;
2065
2066                         if (flow_isolate_all) {
2067                                 int ret = port_flow_isolate(pi, 1);
2068                                 if (ret) {
2069                                         printf("Failed to apply isolated"
2070                                                " mode on port %d\n", pi);
2071                                         return -1;
2072                                 }
2073                         }
2074                         configure_rxtx_dump_callbacks(0);
2075                         printf("Configuring Port %d (socket %u)\n", pi,
2076                                         port->socket_id);
2077                         /* configure port */
2078                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
2079                                                 &(port->dev_conf));
2080                         if (diag != 0) {
2081                                 if (rte_atomic16_cmpset(&(port->port_status),
2082                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2083                                         printf("Port %d can not be set back "
2084                                                         "to stopped\n", pi);
2085                                 printf("Fail to configure port %d\n", pi);
2086                                 /* try to reconfigure port next time */
2087                                 port->need_reconfig = 1;
2088                                 return -1;
2089                         }
2090                 }
2091                 if (port->need_reconfig_queues > 0) {
2092                         port->need_reconfig_queues = 0;
2093                         /* setup tx queues */
2094                         for (qi = 0; qi < nb_txq; qi++) {
2095                                 if ((numa_support) &&
2096                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2097                                         diag = rte_eth_tx_queue_setup(pi, qi,
2098                                                 port->nb_tx_desc[qi],
2099                                                 txring_numa[pi],
2100                                                 &(port->tx_conf[qi]));
2101                                 else
2102                                         diag = rte_eth_tx_queue_setup(pi, qi,
2103                                                 port->nb_tx_desc[qi],
2104                                                 port->socket_id,
2105                                                 &(port->tx_conf[qi]));
2106
2107                                 if (diag == 0)
2108                                         continue;
2109
2110                                 /* Fail to setup tx queue, return */
2111                                 if (rte_atomic16_cmpset(&(port->port_status),
2112                                                         RTE_PORT_HANDLING,
2113                                                         RTE_PORT_STOPPED) == 0)
2114                                         printf("Port %d can not be set back "
2115                                                         "to stopped\n", pi);
2116                                 printf("Fail to configure port %d tx queues\n",
2117                                        pi);
2118                                 /* try to reconfigure queues next time */
2119                                 port->need_reconfig_queues = 1;
2120                                 return -1;
2121                         }
2122                         for (qi = 0; qi < nb_rxq; qi++) {
2123                                 /* setup rx queues */
2124                                 if ((numa_support) &&
2125                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2126                                         struct rte_mempool * mp =
2127                                                 mbuf_pool_find(rxring_numa[pi]);
2128                                         if (mp == NULL) {
2129                                                 printf("Failed to setup RX queue:"
2130                                                         "No mempool allocation"
2131                                                         " on the socket %d\n",
2132                                                         rxring_numa[pi]);
2133                                                 return -1;
2134                                         }
2135
2136                                         diag = rte_eth_rx_queue_setup(pi, qi,
2137                                              port->nb_rx_desc[qi],
2138                                              rxring_numa[pi],
2139                                              &(port->rx_conf[qi]),
2140                                              mp);
2141                                 } else {
2142                                         struct rte_mempool *mp =
2143                                                 mbuf_pool_find(port->socket_id);
2144                                         if (mp == NULL) {
2145                                                 printf("Failed to setup RX queue:"
2146                                                         "No mempool allocation"
2147                                                         " on the socket %d\n",
2148                                                         port->socket_id);
2149                                                 return -1;
2150                                         }
2151                                         diag = rte_eth_rx_queue_setup(pi, qi,
2152                                              port->nb_rx_desc[qi],
2153                                              port->socket_id,
2154                                              &(port->rx_conf[qi]),
2155                                              mp);
2156                                 }
2157                                 if (diag == 0)
2158                                         continue;
2159
2160                                 /* Fail to setup rx queue, return */
2161                                 if (rte_atomic16_cmpset(&(port->port_status),
2162                                                         RTE_PORT_HANDLING,
2163                                                         RTE_PORT_STOPPED) == 0)
2164                                         printf("Port %d can not be set back "
2165                                                         "to stopped\n", pi);
2166                                 printf("Fail to configure port %d rx queues\n",
2167                                        pi);
2168                                 /* try to reconfigure queues next time */
2169                                 port->need_reconfig_queues = 1;
2170                                 return -1;
2171                         }
2172                 }
2173                 configure_rxtx_dump_callbacks(verbose_level);
2174                 /* start port */
2175                 if (rte_eth_dev_start(pi) < 0) {
2176                         printf("Fail to start port %d\n", pi);
2177
2178                         /* Fail to setup rx queue, return */
2179                         if (rte_atomic16_cmpset(&(port->port_status),
2180                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2181                                 printf("Port %d can not be set back to "
2182                                                         "stopped\n", pi);
2183                         continue;
2184                 }
2185
2186                 if (rte_atomic16_cmpset(&(port->port_status),
2187                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2188                         printf("Port %d can not be set into started\n", pi);
2189
2190                 rte_eth_macaddr_get(pi, &mac_addr);
2191                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2192                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2193                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2194                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2195
2196                 /* at least one port started, need checking link status */
2197                 need_check_link_status = 1;
2198         }
2199
2200         if (need_check_link_status == 1 && !no_link_check)
2201                 check_all_ports_link_status(RTE_PORT_ALL);
2202         else if (need_check_link_status == 0)
2203                 printf("Please stop the ports first\n");
2204
2205         printf("Done\n");
2206         return 0;
2207 }
2208
2209 void
2210 stop_port(portid_t pid)
2211 {
2212         portid_t pi;
2213         struct rte_port *port;
2214         int need_check_link_status = 0;
2215
2216         if (dcb_test) {
2217                 dcb_test = 0;
2218                 dcb_config = 0;
2219         }
2220
2221         if (port_id_is_invalid(pid, ENABLED_WARN))
2222                 return;
2223
2224         printf("Stopping ports...\n");
2225
2226         RTE_ETH_FOREACH_DEV(pi) {
2227                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2228                         continue;
2229
2230                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2231                         printf("Please remove port %d from forwarding configuration.\n", pi);
2232                         continue;
2233                 }
2234
2235                 if (port_is_bonding_slave(pi)) {
2236                         printf("Please remove port %d from bonded device.\n", pi);
2237                         continue;
2238                 }
2239
2240                 port = &ports[pi];
2241                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2242                                                 RTE_PORT_HANDLING) == 0)
2243                         continue;
2244
2245                 rte_eth_dev_stop(pi);
2246
2247                 if (rte_atomic16_cmpset(&(port->port_status),
2248                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2249                         printf("Port %d can not be set into stopped\n", pi);
2250                 need_check_link_status = 1;
2251         }
2252         if (need_check_link_status && !no_link_check)
2253                 check_all_ports_link_status(RTE_PORT_ALL);
2254
2255         printf("Done\n");
2256 }
2257
2258 static void
2259 remove_invalid_ports_in(portid_t *array, portid_t *total)
2260 {
2261         portid_t i;
2262         portid_t new_total = 0;
2263
2264         for (i = 0; i < *total; i++)
2265                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2266                         array[new_total] = array[i];
2267                         new_total++;
2268                 }
2269         *total = new_total;
2270 }
2271
2272 static void
2273 remove_invalid_ports(void)
2274 {
2275         remove_invalid_ports_in(ports_ids, &nb_ports);
2276         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2277         nb_cfg_ports = nb_fwd_ports;
2278 }
2279
2280 void
2281 close_port(portid_t pid)
2282 {
2283         portid_t pi;
2284         struct rte_port *port;
2285
2286         if (port_id_is_invalid(pid, ENABLED_WARN))
2287                 return;
2288
2289         printf("Closing ports...\n");
2290
2291         RTE_ETH_FOREACH_DEV(pi) {
2292                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2293                         continue;
2294
2295                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2296                         printf("Please remove port %d from forwarding configuration.\n", pi);
2297                         continue;
2298                 }
2299
2300                 if (port_is_bonding_slave(pi)) {
2301                         printf("Please remove port %d from bonded device.\n", pi);
2302                         continue;
2303                 }
2304
2305                 port = &ports[pi];
2306                 if (rte_atomic16_cmpset(&(port->port_status),
2307                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2308                         printf("Port %d is already closed\n", pi);
2309                         continue;
2310                 }
2311
2312                 if (rte_atomic16_cmpset(&(port->port_status),
2313                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2314                         printf("Port %d is now not stopped\n", pi);
2315                         continue;
2316                 }
2317
2318                 if (port->flow_list)
2319                         port_flow_flush(pi);
2320                 rte_eth_dev_close(pi);
2321
2322                 remove_invalid_ports();
2323
2324                 if (rte_atomic16_cmpset(&(port->port_status),
2325                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2326                         printf("Port %d cannot be set to closed\n", pi);
2327         }
2328
2329         printf("Done\n");
2330 }
2331
2332 void
2333 reset_port(portid_t pid)
2334 {
2335         int diag;
2336         portid_t pi;
2337         struct rte_port *port;
2338
2339         if (port_id_is_invalid(pid, ENABLED_WARN))
2340                 return;
2341
2342         printf("Resetting ports...\n");
2343
2344         RTE_ETH_FOREACH_DEV(pi) {
2345                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2346                         continue;
2347
2348                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2349                         printf("Please remove port %d from forwarding "
2350                                "configuration.\n", pi);
2351                         continue;
2352                 }
2353
2354                 if (port_is_bonding_slave(pi)) {
2355                         printf("Please remove port %d from bonded device.\n",
2356                                pi);
2357                         continue;
2358                 }
2359
2360                 diag = rte_eth_dev_reset(pi);
2361                 if (diag == 0) {
2362                         port = &ports[pi];
2363                         port->need_reconfig = 1;
2364                         port->need_reconfig_queues = 1;
2365                 } else {
2366                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2367                 }
2368         }
2369
2370         printf("Done\n");
2371 }
2372
2373 void
2374 attach_port(char *identifier)
2375 {
2376         portid_t pi;
2377         struct rte_dev_iterator iterator;
2378
2379         printf("Attaching a new port...\n");
2380
2381         if (identifier == NULL) {
2382                 printf("Invalid parameters are specified\n");
2383                 return;
2384         }
2385
2386         if (rte_dev_probe(identifier) < 0) {
2387                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2388                 return;
2389         }
2390
2391         /* first attach mode: event */
2392         if (setup_on_probe_event) {
2393                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2394                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2395                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2396                                         ports[pi].need_setup != 0)
2397                                 setup_attached_port(pi);
2398                 return;
2399         }
2400
2401         /* second attach mode: iterator */
2402         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2403                 /* setup ports matching the devargs used for probing */
2404                 if (port_is_forwarding(pi))
2405                         continue; /* port was already attached before */
2406                 setup_attached_port(pi);
2407         }
2408 }
2409
2410 static void
2411 setup_attached_port(portid_t pi)
2412 {
2413         unsigned int socket_id;
2414
2415         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2416         /* if socket_id is invalid, set to the first available socket. */
2417         if (check_socket_id(socket_id) < 0)
2418                 socket_id = socket_ids[0];
2419         reconfig(pi, socket_id);
2420         rte_eth_promiscuous_enable(pi);
2421
2422         ports_ids[nb_ports++] = pi;
2423         fwd_ports_ids[nb_fwd_ports++] = pi;
2424         nb_cfg_ports = nb_fwd_ports;
2425         ports[pi].need_setup = 0;
2426         ports[pi].port_status = RTE_PORT_STOPPED;
2427
2428         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2429         printf("Done\n");
2430 }
2431
2432 void
2433 detach_port_device(portid_t port_id)
2434 {
2435         struct rte_device *dev;
2436         portid_t sibling;
2437
2438         printf("Removing a device...\n");
2439
2440         dev = rte_eth_devices[port_id].device;
2441         if (dev == NULL) {
2442                 printf("Device already removed\n");
2443                 return;
2444         }
2445
2446         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2447                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2448                         printf("Port not stopped\n");
2449                         return;
2450                 }
2451                 printf("Port was not closed\n");
2452                 if (ports[port_id].flow_list)
2453                         port_flow_flush(port_id);
2454         }
2455
2456         if (rte_dev_remove(dev) < 0) {
2457                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2458                 return;
2459         }
2460         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2461                 /* reset mapping between old ports and removed device */
2462                 rte_eth_devices[sibling].device = NULL;
2463                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2464                         /* sibling ports are forced to be closed */
2465                         ports[sibling].port_status = RTE_PORT_CLOSED;
2466                         printf("Port %u is closed\n", sibling);
2467                 }
2468         }
2469
2470         remove_invalid_ports();
2471
2472         printf("Device of port %u is detached\n", port_id);
2473         printf("Now total ports is %d\n", nb_ports);
2474         printf("Done\n");
2475         return;
2476 }
2477
2478 void
2479 detach_device(char *identifier)
2480 {
2481         struct rte_dev_iterator iterator;
2482         struct rte_devargs da;
2483         portid_t port_id;
2484
2485         printf("Removing a device...\n");
2486
2487         memset(&da, 0, sizeof(da));
2488         if (rte_devargs_parsef(&da, "%s", identifier)) {
2489                 printf("cannot parse identifier\n");
2490                 if (da.args)
2491                         free(da.args);
2492                 return;
2493         }
2494
2495         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2496                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2497                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2498                                 printf("Port %u not stopped\n", port_id);
2499                                 return;
2500                         }
2501
2502                         /* sibling ports are forced to be closed */
2503                         if (ports[port_id].flow_list)
2504                                 port_flow_flush(port_id);
2505                         ports[port_id].port_status = RTE_PORT_CLOSED;
2506                         printf("Port %u is now closed\n", port_id);
2507                 }
2508         }
2509
2510         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2511                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2512                             da.name, da.bus->name);
2513                 return;
2514         }
2515
2516         remove_invalid_ports();
2517
2518         printf("Device %s is detached\n", identifier);
2519         printf("Now total ports is %d\n", nb_ports);
2520         printf("Done\n");
2521 }
2522
2523 void
2524 pmd_test_exit(void)
2525 {
2526         portid_t pt_id;
2527         int ret;
2528         int i;
2529
2530         if (test_done == 0)
2531                 stop_packet_forwarding();
2532
2533         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2534                 if (mempools[i]) {
2535                         if (mp_alloc_type == MP_ALLOC_ANON)
2536                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2537                                                      NULL);
2538                 }
2539         }
2540         if (ports != NULL) {
2541                 no_link_check = 1;
2542                 RTE_ETH_FOREACH_DEV(pt_id) {
2543                         printf("\nStopping port %d...\n", pt_id);
2544                         fflush(stdout);
2545                         stop_port(pt_id);
2546                 }
2547                 RTE_ETH_FOREACH_DEV(pt_id) {
2548                         printf("\nShutting down port %d...\n", pt_id);
2549                         fflush(stdout);
2550                         close_port(pt_id);
2551                 }
2552         }
2553
2554         if (hot_plug) {
2555                 ret = rte_dev_event_monitor_stop();
2556                 if (ret) {
2557                         RTE_LOG(ERR, EAL,
2558                                 "fail to stop device event monitor.");
2559                         return;
2560                 }
2561
2562                 ret = rte_dev_event_callback_unregister(NULL,
2563                         dev_event_callback, NULL);
2564                 if (ret < 0) {
2565                         RTE_LOG(ERR, EAL,
2566                                 "fail to unregister device event callback.\n");
2567                         return;
2568                 }
2569
2570                 ret = rte_dev_hotplug_handle_disable();
2571                 if (ret) {
2572                         RTE_LOG(ERR, EAL,
2573                                 "fail to disable hotplug handling.\n");
2574                         return;
2575                 }
2576         }
2577         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2578                 if (mempools[i])
2579                         rte_mempool_free(mempools[i]);
2580         }
2581
2582         printf("\nBye...\n");
2583 }
2584
2585 typedef void (*cmd_func_t)(void);
2586 struct pmd_test_command {
2587         const char *cmd_name;
2588         cmd_func_t cmd_func;
2589 };
2590
2591 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2592
2593 /* Check the link status of all ports in up to 9s, and print them finally */
2594 static void
2595 check_all_ports_link_status(uint32_t port_mask)
2596 {
2597 #define CHECK_INTERVAL 100 /* 100ms */
2598 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2599         portid_t portid;
2600         uint8_t count, all_ports_up, print_flag = 0;
2601         struct rte_eth_link link;
2602
2603         printf("Checking link statuses...\n");
2604         fflush(stdout);
2605         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2606                 all_ports_up = 1;
2607                 RTE_ETH_FOREACH_DEV(portid) {
2608                         if ((port_mask & (1 << portid)) == 0)
2609                                 continue;
2610                         memset(&link, 0, sizeof(link));
2611                         rte_eth_link_get_nowait(portid, &link);
2612                         /* print link status if flag set */
2613                         if (print_flag == 1) {
2614                                 if (link.link_status)
2615                                         printf(
2616                                         "Port%d Link Up. speed %u Mbps- %s\n",
2617                                         portid, link.link_speed,
2618                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2619                                         ("full-duplex") : ("half-duplex\n"));
2620                                 else
2621                                         printf("Port %d Link Down\n", portid);
2622                                 continue;
2623                         }
2624                         /* clear all_ports_up flag if any link down */
2625                         if (link.link_status == ETH_LINK_DOWN) {
2626                                 all_ports_up = 0;
2627                                 break;
2628                         }
2629                 }
2630                 /* after finally printing all link status, get out */
2631                 if (print_flag == 1)
2632                         break;
2633
2634                 if (all_ports_up == 0) {
2635                         fflush(stdout);
2636                         rte_delay_ms(CHECK_INTERVAL);
2637                 }
2638
2639                 /* set the print_flag if all ports up or timeout */
2640                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2641                         print_flag = 1;
2642                 }
2643
2644                 if (lsc_interrupt)
2645                         break;
2646         }
2647 }
2648
2649 /*
2650  * This callback is for remove a port for a device. It has limitation because
2651  * it is not for multiple port removal for a device.
2652  * TODO: the device detach invoke will plan to be removed from user side to
2653  * eal. And convert all PMDs to free port resources on ether device closing.
2654  */
2655 static void
2656 rmv_port_callback(void *arg)
2657 {
2658         int need_to_start = 0;
2659         int org_no_link_check = no_link_check;
2660         portid_t port_id = (intptr_t)arg;
2661
2662         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2663
2664         if (!test_done && port_is_forwarding(port_id)) {
2665                 need_to_start = 1;
2666                 stop_packet_forwarding();
2667         }
2668         no_link_check = 1;
2669         stop_port(port_id);
2670         no_link_check = org_no_link_check;
2671         close_port(port_id);
2672         detach_port_device(port_id);
2673         if (need_to_start)
2674                 start_packet_forwarding(0);
2675 }
2676
2677 /* This function is used by the interrupt thread */
2678 static int
2679 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2680                   void *ret_param)
2681 {
2682         RTE_SET_USED(param);
2683         RTE_SET_USED(ret_param);
2684
2685         if (type >= RTE_ETH_EVENT_MAX) {
2686                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2687                         port_id, __func__, type);
2688                 fflush(stderr);
2689         } else if (event_print_mask & (UINT32_C(1) << type)) {
2690                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2691                         eth_event_desc[type]);
2692                 fflush(stdout);
2693         }
2694
2695         switch (type) {
2696         case RTE_ETH_EVENT_NEW:
2697                 ports[port_id].need_setup = 1;
2698                 ports[port_id].port_status = RTE_PORT_HANDLING;
2699                 break;
2700         case RTE_ETH_EVENT_INTR_RMV:
2701                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2702                         break;
2703                 if (rte_eal_alarm_set(100000,
2704                                 rmv_port_callback, (void *)(intptr_t)port_id))
2705                         fprintf(stderr, "Could not set up deferred device removal\n");
2706                 break;
2707         default:
2708                 break;
2709         }
2710         return 0;
2711 }
2712
2713 static int
2714 register_eth_event_callback(void)
2715 {
2716         int ret;
2717         enum rte_eth_event_type event;
2718
2719         for (event = RTE_ETH_EVENT_UNKNOWN;
2720                         event < RTE_ETH_EVENT_MAX; event++) {
2721                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2722                                 event,
2723                                 eth_event_callback,
2724                                 NULL);
2725                 if (ret != 0) {
2726                         TESTPMD_LOG(ERR, "Failed to register callback for "
2727                                         "%s event\n", eth_event_desc[event]);
2728                         return -1;
2729                 }
2730         }
2731
2732         return 0;
2733 }
2734
2735 /* This function is used by the interrupt thread */
2736 static void
2737 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2738                              __rte_unused void *arg)
2739 {
2740         uint16_t port_id;
2741         int ret;
2742
2743         if (type >= RTE_DEV_EVENT_MAX) {
2744                 fprintf(stderr, "%s called upon invalid event %d\n",
2745                         __func__, type);
2746                 fflush(stderr);
2747         }
2748
2749         switch (type) {
2750         case RTE_DEV_EVENT_REMOVE:
2751                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2752                         device_name);
2753                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2754                 if (ret) {
2755                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2756                                 device_name);
2757                         return;
2758                 }
2759                 /*
2760                  * Because the user's callback is invoked in eal interrupt
2761                  * callback, the interrupt callback need to be finished before
2762                  * it can be unregistered when detaching device. So finish
2763                  * callback soon and use a deferred removal to detach device
2764                  * is need. It is a workaround, once the device detaching be
2765                  * moved into the eal in the future, the deferred removal could
2766                  * be deleted.
2767                  */
2768                 if (rte_eal_alarm_set(100000,
2769                                 rmv_port_callback, (void *)(intptr_t)port_id))
2770                         RTE_LOG(ERR, EAL,
2771                                 "Could not set up deferred device removal\n");
2772                 break;
2773         case RTE_DEV_EVENT_ADD:
2774                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2775                         device_name);
2776                 /* TODO: After finish kernel driver binding,
2777                  * begin to attach port.
2778                  */
2779                 break;
2780         default:
2781                 break;
2782         }
2783 }
2784
2785 static int
2786 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2787 {
2788         uint16_t i;
2789         int diag;
2790         uint8_t mapping_found = 0;
2791
2792         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2793                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2794                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2795                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2796                                         tx_queue_stats_mappings[i].queue_id,
2797                                         tx_queue_stats_mappings[i].stats_counter_id);
2798                         if (diag != 0)
2799                                 return diag;
2800                         mapping_found = 1;
2801                 }
2802         }
2803         if (mapping_found)
2804                 port->tx_queue_stats_mapping_enabled = 1;
2805         return 0;
2806 }
2807
2808 static int
2809 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2810 {
2811         uint16_t i;
2812         int diag;
2813         uint8_t mapping_found = 0;
2814
2815         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2816                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2817                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2818                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2819                                         rx_queue_stats_mappings[i].queue_id,
2820                                         rx_queue_stats_mappings[i].stats_counter_id);
2821                         if (diag != 0)
2822                                 return diag;
2823                         mapping_found = 1;
2824                 }
2825         }
2826         if (mapping_found)
2827                 port->rx_queue_stats_mapping_enabled = 1;
2828         return 0;
2829 }
2830
2831 static void
2832 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2833 {
2834         int diag = 0;
2835
2836         diag = set_tx_queue_stats_mapping_registers(pi, port);
2837         if (diag != 0) {
2838                 if (diag == -ENOTSUP) {
2839                         port->tx_queue_stats_mapping_enabled = 0;
2840                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2841                 }
2842                 else
2843                         rte_exit(EXIT_FAILURE,
2844                                         "set_tx_queue_stats_mapping_registers "
2845                                         "failed for port id=%d diag=%d\n",
2846                                         pi, diag);
2847         }
2848
2849         diag = set_rx_queue_stats_mapping_registers(pi, port);
2850         if (diag != 0) {
2851                 if (diag == -ENOTSUP) {
2852                         port->rx_queue_stats_mapping_enabled = 0;
2853                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2854                 }
2855                 else
2856                         rte_exit(EXIT_FAILURE,
2857                                         "set_rx_queue_stats_mapping_registers "
2858                                         "failed for port id=%d diag=%d\n",
2859                                         pi, diag);
2860         }
2861 }
2862
2863 static void
2864 rxtx_port_config(struct rte_port *port)
2865 {
2866         uint16_t qid;
2867         uint64_t offloads;
2868
2869         for (qid = 0; qid < nb_rxq; qid++) {
2870                 offloads = port->rx_conf[qid].offloads;
2871                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2872                 if (offloads != 0)
2873                         port->rx_conf[qid].offloads = offloads;
2874
2875                 /* Check if any Rx parameters have been passed */
2876                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2877                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2878
2879                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2880                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2881
2882                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2883                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2884
2885                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2886                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2887
2888                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2889                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2890
2891                 port->nb_rx_desc[qid] = nb_rxd;
2892         }
2893
2894         for (qid = 0; qid < nb_txq; qid++) {
2895                 offloads = port->tx_conf[qid].offloads;
2896                 port->tx_conf[qid] = port->dev_info.default_txconf;
2897                 if (offloads != 0)
2898                         port->tx_conf[qid].offloads = offloads;
2899
2900                 /* Check if any Tx parameters have been passed */
2901                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2902                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2903
2904                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2905                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2906
2907                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2908                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2909
2910                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2911                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2912
2913                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2914                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2915
2916                 port->nb_tx_desc[qid] = nb_txd;
2917         }
2918 }
2919
2920 void
2921 init_port_config(void)
2922 {
2923         portid_t pid;
2924         struct rte_port *port;
2925
2926         RTE_ETH_FOREACH_DEV(pid) {
2927                 port = &ports[pid];
2928                 port->dev_conf.fdir_conf = fdir_conf;
2929                 rte_eth_dev_info_get(pid, &port->dev_info);
2930                 if (nb_rxq > 1) {
2931                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2932                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2933                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2934                 } else {
2935                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2936                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2937                 }
2938
2939                 if (port->dcb_flag == 0) {
2940                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2941                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2942                         else
2943                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2944                 }
2945
2946                 rxtx_port_config(port);
2947
2948                 rte_eth_macaddr_get(pid, &port->eth_addr);
2949
2950                 map_port_queue_stats_mapping_registers(pid, port);
2951 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2952                 rte_pmd_ixgbe_bypass_init(pid);
2953 #endif
2954
2955                 if (lsc_interrupt &&
2956                     (rte_eth_devices[pid].data->dev_flags &
2957                      RTE_ETH_DEV_INTR_LSC))
2958                         port->dev_conf.intr_conf.lsc = 1;
2959                 if (rmv_interrupt &&
2960                     (rte_eth_devices[pid].data->dev_flags &
2961                      RTE_ETH_DEV_INTR_RMV))
2962                         port->dev_conf.intr_conf.rmv = 1;
2963         }
2964 }
2965
2966 void set_port_slave_flag(portid_t slave_pid)
2967 {
2968         struct rte_port *port;
2969
2970         port = &ports[slave_pid];
2971         port->slave_flag = 1;
2972 }
2973
2974 void clear_port_slave_flag(portid_t slave_pid)
2975 {
2976         struct rte_port *port;
2977
2978         port = &ports[slave_pid];
2979         port->slave_flag = 0;
2980 }
2981
2982 uint8_t port_is_bonding_slave(portid_t slave_pid)
2983 {
2984         struct rte_port *port;
2985
2986         port = &ports[slave_pid];
2987         if ((rte_eth_devices[slave_pid].data->dev_flags &
2988             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
2989                 return 1;
2990         return 0;
2991 }
2992
2993 const uint16_t vlan_tags[] = {
2994                 0,  1,  2,  3,  4,  5,  6,  7,
2995                 8,  9, 10, 11,  12, 13, 14, 15,
2996                 16, 17, 18, 19, 20, 21, 22, 23,
2997                 24, 25, 26, 27, 28, 29, 30, 31
2998 };
2999
3000 static  int
3001 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3002                  enum dcb_mode_enable dcb_mode,
3003                  enum rte_eth_nb_tcs num_tcs,
3004                  uint8_t pfc_en)
3005 {
3006         uint8_t i;
3007         int32_t rc;
3008         struct rte_eth_rss_conf rss_conf;
3009
3010         /*
3011          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3012          * given above, and the number of traffic classes available for use.
3013          */
3014         if (dcb_mode == DCB_VT_ENABLED) {
3015                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3016                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3017                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3018                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3019
3020                 /* VMDQ+DCB RX and TX configurations */
3021                 vmdq_rx_conf->enable_default_pool = 0;
3022                 vmdq_rx_conf->default_pool = 0;
3023                 vmdq_rx_conf->nb_queue_pools =
3024                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3025                 vmdq_tx_conf->nb_queue_pools =
3026                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3027
3028                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3029                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3030                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3031                         vmdq_rx_conf->pool_map[i].pools =
3032                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3033                 }
3034                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3035                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3036                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3037                 }
3038
3039                 /* set DCB mode of RX and TX of multiple queues */
3040                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
3041                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3042         } else {
3043                 struct rte_eth_dcb_rx_conf *rx_conf =
3044                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3045                 struct rte_eth_dcb_tx_conf *tx_conf =
3046                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3047
3048                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3049                 if (rc != 0)
3050                         return rc;
3051
3052                 rx_conf->nb_tcs = num_tcs;
3053                 tx_conf->nb_tcs = num_tcs;
3054
3055                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3056                         rx_conf->dcb_tc[i] = i % num_tcs;
3057                         tx_conf->dcb_tc[i] = i % num_tcs;
3058                 }
3059
3060                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
3061                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3062                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3063         }
3064
3065         if (pfc_en)
3066                 eth_conf->dcb_capability_en =
3067                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3068         else
3069                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3070
3071         return 0;
3072 }
3073
3074 int
3075 init_port_dcb_config(portid_t pid,
3076                      enum dcb_mode_enable dcb_mode,
3077                      enum rte_eth_nb_tcs num_tcs,
3078                      uint8_t pfc_en)
3079 {
3080         struct rte_eth_conf port_conf;
3081         struct rte_port *rte_port;
3082         int retval;
3083         uint16_t i;
3084
3085         rte_port = &ports[pid];
3086
3087         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3088         /* Enter DCB configuration status */
3089         dcb_config = 1;
3090
3091         port_conf.rxmode = rte_port->dev_conf.rxmode;
3092         port_conf.txmode = rte_port->dev_conf.txmode;
3093
3094         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3095         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3096         if (retval < 0)
3097                 return retval;
3098         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3099
3100         /* re-configure the device . */
3101         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3102         if (retval < 0)
3103                 return retval;
3104         rte_eth_dev_info_get(pid, &rte_port->dev_info);
3105
3106         /* If dev_info.vmdq_pool_base is greater than 0,
3107          * the queue id of vmdq pools is started after pf queues.
3108          */
3109         if (dcb_mode == DCB_VT_ENABLED &&
3110             rte_port->dev_info.vmdq_pool_base > 0) {
3111                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3112                         " for port %d.", pid);
3113                 return -1;
3114         }
3115
3116         /* Assume the ports in testpmd have the same dcb capability
3117          * and has the same number of rxq and txq in dcb mode
3118          */
3119         if (dcb_mode == DCB_VT_ENABLED) {
3120                 if (rte_port->dev_info.max_vfs > 0) {
3121                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3122                         nb_txq = rte_port->dev_info.nb_tx_queues;
3123                 } else {
3124                         nb_rxq = rte_port->dev_info.max_rx_queues;
3125                         nb_txq = rte_port->dev_info.max_tx_queues;
3126                 }
3127         } else {
3128                 /*if vt is disabled, use all pf queues */
3129                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3130                         nb_rxq = rte_port->dev_info.max_rx_queues;
3131                         nb_txq = rte_port->dev_info.max_tx_queues;
3132                 } else {
3133                         nb_rxq = (queueid_t)num_tcs;
3134                         nb_txq = (queueid_t)num_tcs;
3135
3136                 }
3137         }
3138         rx_free_thresh = 64;
3139
3140         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3141
3142         rxtx_port_config(rte_port);
3143         /* VLAN filter */
3144         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3145         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3146                 rx_vft_set(pid, vlan_tags[i], 1);
3147
3148         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
3149         map_port_queue_stats_mapping_registers(pid, rte_port);
3150
3151         rte_port->dcb_flag = 1;
3152
3153         return 0;
3154 }
3155
3156 static void
3157 init_port(void)
3158 {
3159         /* Configuration of Ethernet ports. */
3160         ports = rte_zmalloc("testpmd: ports",
3161                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3162                             RTE_CACHE_LINE_SIZE);
3163         if (ports == NULL) {
3164                 rte_exit(EXIT_FAILURE,
3165                                 "rte_zmalloc(%d struct rte_port) failed\n",
3166                                 RTE_MAX_ETHPORTS);
3167         }
3168
3169         /* Initialize ports NUMA structures */
3170         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3171         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3172         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3173 }
3174
3175 static void
3176 force_quit(void)
3177 {
3178         pmd_test_exit();
3179         prompt_exit();
3180 }
3181
3182 static void
3183 print_stats(void)
3184 {
3185         uint8_t i;
3186         const char clr[] = { 27, '[', '2', 'J', '\0' };
3187         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3188
3189         /* Clear screen and move to top left */
3190         printf("%s%s", clr, top_left);
3191
3192         printf("\nPort statistics ====================================");
3193         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3194                 nic_stats_display(fwd_ports_ids[i]);
3195
3196         fflush(stdout);
3197 }
3198
3199 static void
3200 signal_handler(int signum)
3201 {
3202         if (signum == SIGINT || signum == SIGTERM) {
3203                 printf("\nSignal %d received, preparing to exit...\n",
3204                                 signum);
3205 #ifdef RTE_LIBRTE_PDUMP
3206                 /* uninitialize packet capture framework */
3207                 rte_pdump_uninit();
3208 #endif
3209 #ifdef RTE_LIBRTE_LATENCY_STATS
3210                 rte_latencystats_uninit();
3211 #endif
3212                 force_quit();
3213                 /* Set flag to indicate the force termination. */
3214                 f_quit = 1;
3215                 /* exit with the expected status */
3216                 signal(signum, SIG_DFL);
3217                 kill(getpid(), signum);
3218         }
3219 }
3220
3221 int
3222 main(int argc, char** argv)
3223 {
3224         int diag;
3225         portid_t port_id;
3226         uint16_t count;
3227         int ret;
3228
3229         signal(SIGINT, signal_handler);
3230         signal(SIGTERM, signal_handler);
3231
3232         diag = rte_eal_init(argc, argv);
3233         if (diag < 0)
3234                 rte_panic("Cannot init EAL\n");
3235
3236         testpmd_logtype = rte_log_register("testpmd");
3237         if (testpmd_logtype < 0)
3238                 rte_panic("Cannot register log type");
3239         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3240
3241         ret = register_eth_event_callback();
3242         if (ret != 0)
3243                 rte_panic("Cannot register for ethdev events");
3244
3245 #ifdef RTE_LIBRTE_PDUMP
3246         /* initialize packet capture framework */
3247         rte_pdump_init();
3248 #endif
3249
3250         count = 0;
3251         RTE_ETH_FOREACH_DEV(port_id) {
3252                 ports_ids[count] = port_id;
3253                 count++;
3254         }
3255         nb_ports = (portid_t) count;
3256         if (nb_ports == 0)
3257                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3258
3259         /* allocate port structures, and init them */
3260         init_port();
3261
3262         set_def_fwd_config();
3263         if (nb_lcores == 0)
3264                 rte_panic("Empty set of forwarding logical cores - check the "
3265                           "core mask supplied in the command parameters\n");
3266
3267         /* Bitrate/latency stats disabled by default */
3268 #ifdef RTE_LIBRTE_BITRATE
3269         bitrate_enabled = 0;
3270 #endif
3271 #ifdef RTE_LIBRTE_LATENCY_STATS
3272         latencystats_enabled = 0;
3273 #endif
3274
3275         /* on FreeBSD, mlockall() is disabled by default */
3276 #ifdef RTE_EXEC_ENV_FREEBSD
3277         do_mlockall = 0;
3278 #else
3279         do_mlockall = 1;
3280 #endif
3281
3282         argc -= diag;
3283         argv += diag;
3284         if (argc > 1)
3285                 launch_args_parse(argc, argv);
3286
3287         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3288                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3289                         strerror(errno));
3290         }
3291
3292         if (tx_first && interactive)
3293                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3294                                 "interactive mode.\n");
3295
3296         if (tx_first && lsc_interrupt) {
3297                 printf("Warning: lsc_interrupt needs to be off when "
3298                                 " using tx_first. Disabling.\n");
3299                 lsc_interrupt = 0;
3300         }
3301
3302         if (!nb_rxq && !nb_txq)
3303                 printf("Warning: Either rx or tx queues should be non-zero\n");
3304
3305         if (nb_rxq > 1 && nb_rxq > nb_txq)
3306                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3307                        "but nb_txq=%d will prevent to fully test it.\n",
3308                        nb_rxq, nb_txq);
3309
3310         init_config();
3311
3312         if (hot_plug) {
3313                 ret = rte_dev_hotplug_handle_enable();
3314                 if (ret) {
3315                         RTE_LOG(ERR, EAL,
3316                                 "fail to enable hotplug handling.");
3317                         return -1;
3318                 }
3319
3320                 ret = rte_dev_event_monitor_start();
3321                 if (ret) {
3322                         RTE_LOG(ERR, EAL,
3323                                 "fail to start device event monitoring.");
3324                         return -1;
3325                 }
3326
3327                 ret = rte_dev_event_callback_register(NULL,
3328                         dev_event_callback, NULL);
3329                 if (ret) {
3330                         RTE_LOG(ERR, EAL,
3331                                 "fail  to register device event callback\n");
3332                         return -1;
3333                 }
3334         }
3335
3336         if (start_port(RTE_PORT_ALL) != 0)
3337                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3338
3339         /* set all ports to promiscuous mode by default */
3340         RTE_ETH_FOREACH_DEV(port_id)
3341                 rte_eth_promiscuous_enable(port_id);
3342
3343         /* Init metrics library */
3344         rte_metrics_init(rte_socket_id());
3345
3346 #ifdef RTE_LIBRTE_LATENCY_STATS
3347         if (latencystats_enabled != 0) {
3348                 int ret = rte_latencystats_init(1, NULL);
3349                 if (ret)
3350                         printf("Warning: latencystats init()"
3351                                 " returned error %d\n", ret);
3352                 printf("Latencystats running on lcore %d\n",
3353                         latencystats_lcore_id);
3354         }
3355 #endif
3356
3357         /* Setup bitrate stats */
3358 #ifdef RTE_LIBRTE_BITRATE
3359         if (bitrate_enabled != 0) {
3360                 bitrate_data = rte_stats_bitrate_create();
3361                 if (bitrate_data == NULL)
3362                         rte_exit(EXIT_FAILURE,
3363                                 "Could not allocate bitrate data.\n");
3364                 rte_stats_bitrate_reg(bitrate_data);
3365         }
3366 #endif
3367
3368 #ifdef RTE_LIBRTE_CMDLINE
3369         if (strlen(cmdline_filename) != 0)
3370                 cmdline_read_from_file(cmdline_filename);
3371
3372         if (interactive == 1) {
3373                 if (auto_start) {
3374                         printf("Start automatic packet forwarding\n");
3375                         start_packet_forwarding(0);
3376                 }
3377                 prompt();
3378                 pmd_test_exit();
3379         } else
3380 #endif
3381         {
3382                 char c;
3383                 int rc;
3384
3385                 f_quit = 0;
3386
3387                 printf("No commandline core given, start packet forwarding\n");
3388                 start_packet_forwarding(tx_first);
3389                 if (stats_period != 0) {
3390                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3391                         uint64_t timer_period;
3392
3393                         /* Convert to number of cycles */
3394                         timer_period = stats_period * rte_get_timer_hz();
3395
3396                         while (f_quit == 0) {
3397                                 cur_time = rte_get_timer_cycles();
3398                                 diff_time += cur_time - prev_time;
3399
3400                                 if (diff_time >= timer_period) {
3401                                         print_stats();
3402                                         /* Reset the timer */
3403                                         diff_time = 0;
3404                                 }
3405                                 /* Sleep to avoid unnecessary checks */
3406                                 prev_time = cur_time;
3407                                 sleep(1);
3408                         }
3409                 }
3410
3411                 printf("Press enter to exit\n");
3412                 rc = read(0, &c, 1);
3413                 pmd_test_exit();
3414                 if (rc < 0)
3415                         return 1;
3416         }
3417
3418         return 0;
3419 }