app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_malloc_heap.h>
  31 #include <rte_memory.h>
  32 #include <rte_memcpy.h>
  33 #include <rte_launch.h>
  34 #include <rte_eal.h>
  35 #include <rte_alarm.h>
  36 #include <rte_per_lcore.h>
  37 #include <rte_lcore.h>
  38 #include <rte_atomic.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_LIBRTE_IXGBE_PMD
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIBRTE_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #include <rte_metrics.h>
  58 #ifdef RTE_LIBRTE_BITRATE
  59 #include <rte_bitrate.h>
  60 #endif
  61 #ifdef RTE_LIBRTE_LATENCY_STATS
  62 #include <rte_latencystats.h>
  63 #endif
  64
  65 #include "testpmd.h"
  66
  67 #ifndef MAP_HUGETLB
  68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  69 #define HUGE_FLAG (0x40000)
  70 #else
  71 #define HUGE_FLAG MAP_HUGETLB
  72 #endif
  73
  74 #ifndef MAP_HUGE_SHIFT
  75 /* older kernels (or FreeBSD) will not have this define */
  76 #define HUGE_SHIFT (26)
  77 #else
  78 #define HUGE_SHIFT MAP_HUGE_SHIFT
  79 #endif
  80
  81 #define EXTMEM_HEAP_NAME "extmem"
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192 uint16_t mempool_flags;
 193
 194 struct fwd_config cur_fwd_config;
 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 196 uint32_t retry_enabled;
 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 199
 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 201 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 202                                       * specified on command-line. */
 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 204
 205 /*
 206  * In container, it cannot terminate the process which running with 'stats-period'
 207  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 208  */
 209 uint8_t f_quit;
 210
 211 /*
 212  * Configuration of packet segments used by the "txonly" processing engine.
 213  */
 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 216         TXONLY_DEF_PACKET_LEN,
 217 };
 218 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 219
 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 221 /**< Split policy for packets to TX. */
 222
 223 uint8_t txonly_multi_flow;
 224 /**< Whether multiple flows are generated in TXONLY mode. */
 225
 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 228
 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 230 uint8_t dcb_config = 0;
 231
 232 /* Whether the dcb is in testing status */
 233 uint8_t dcb_test = 0;
 234
 235 /*
 236  * Configurable number of RX/TX queues.
 237  */
 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 240
 241 /*
 242  * Configurable number of RX/TX ring descriptors.
 243  * Defaults are supplied by drivers via ethdev.
 244  */
 245 #define RTE_TEST_RX_DESC_DEFAULT 0
 246 #define RTE_TEST_TX_DESC_DEFAULT 0
 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 249
 250 #define RTE_PMD_PARAM_UNSET -1
 251 /*
 252  * Configurable values of RX and TX ring threshold registers.
 253  */
 254
 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 258
 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 262
 263 /*
 264  * Configurable value of RX free threshold.
 265  */
 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 267
 268 /*
 269  * Configurable value of RX drop enable.
 270  */
 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 272
 273 /*
 274  * Configurable value of TX free threshold.
 275  */
 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of TX RS bit threshold.
 280  */
 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of buffered packets before sending.
 285  */
 286 uint16_t noisy_tx_sw_bufsz;
 287
 288 /*
 289  * Configurable value of packet buffer timeout.
 290  */
 291 uint16_t noisy_tx_sw_buf_flush_time;
 292
 293 /*
 294  * Configurable value for size of VNF internal memory area
 295  * used for simulating noisy neighbour behaviour
 296  */
 297 uint64_t noisy_lkup_mem_sz;
 298
 299 /*
 300  * Configurable value of number of random writes done in
 301  * VNF simulation memory area.
 302  */
 303 uint64_t noisy_lkup_num_writes;
 304
 305 /*
 306  * Configurable value of number of random reads done in
 307  * VNF simulation memory area.
 308  */
 309 uint64_t noisy_lkup_num_reads;
 310
 311 /*
 312  * Configurable value of number of random reads/writes done in
 313  * VNF simulation memory area.
 314  */
 315 uint64_t noisy_lkup_num_reads_writes;
 316
 317 /*
 318  * Receive Side Scaling (RSS) configuration.
 319  */
 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 321
 322 /*
 323  * Port topology configuration
 324  */
 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 326
 327 /*
 328  * Avoids to flush all the RX streams before starts forwarding.
 329  */
 330 uint8_t no_flush_rx = 0; /* flush by default */
 331
 332 /*
 333  * Flow API isolated mode.
 334  */
 335 uint8_t flow_isolate_all;
 336
 337 /*
 338  * Avoids to check link status when starting/stopping a port.
 339  */
 340 uint8_t no_link_check = 0; /* check by default */
 341
 342 /*
 343  * Enable link status change notification
 344  */
 345 uint8_t lsc_interrupt = 1; /* enabled by default */
 346
 347 /*
 348  * Enable device removal notification.
 349  */
 350 uint8_t rmv_interrupt = 1; /* enabled by default */
 351
 352 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 353
 354 /* After attach, port setup is called on event or by iterator */
 355 bool setup_on_probe_event = true;
 356
 357 /* Pretty printing of ethdev events */
 358 static const char * const eth_event_desc[] = {
 359         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 360         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 361         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 362         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 363         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 364         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 365         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 366         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 367         [RTE_ETH_EVENT_NEW] = "device probed",
 368         [RTE_ETH_EVENT_DESTROY] = "device released",
 369         [RTE_ETH_EVENT_MAX] = NULL,
 370 };
 371
 372 /*
 373  * Display or mask ether events
 374  * Default to all events except VF_MBOX
 375  */
 376 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 377                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 378                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 379                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 380                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 381                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 382                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 383 /*
 384  * Decide if all memory are locked for performance.
 385  */
 386 int do_mlockall = 0;
 387
 388 /*
 389  * NIC bypass mode configuration options.
 390  */
 391
 392 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 393 /* The NIC bypass watchdog timeout. */
 394 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 395 #endif
 396
 397
 398 #ifdef RTE_LIBRTE_LATENCY_STATS
 399
 400 /*
 401  * Set when latency stats is enabled in the commandline
 402  */
 403 uint8_t latencystats_enabled;
 404
 405 /*
 406  * Lcore ID to serive latency statistics.
 407  */
 408 lcoreid_t latencystats_lcore_id = -1;
 409
 410 #endif
 411
 412 /*
 413  * Ethernet device configuration.
 414  */
 415 struct rte_eth_rxmode rx_mode = {
 416         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 417                 /**< Default maximum frame length. */
 418 };
 419
 420 struct rte_eth_txmode tx_mode = {
 421         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 422 };
 423
 424 struct rte_fdir_conf fdir_conf = {
 425         .mode = RTE_FDIR_MODE_NONE,
 426         .pballoc = RTE_FDIR_PBALLOC_64K,
 427         .status = RTE_FDIR_REPORT_STATUS,
 428         .mask = {
 429                 .vlan_tci_mask = 0xFFEF,
 430                 .ipv4_mask     = {
 431                         .src_ip = 0xFFFFFFFF,
 432                         .dst_ip = 0xFFFFFFFF,
 433                 },
 434                 .ipv6_mask     = {
 435                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 436                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 437                 },
 438                 .src_port_mask = 0xFFFF,
 439                 .dst_port_mask = 0xFFFF,
 440                 .mac_addr_byte_mask = 0xFF,
 441                 .tunnel_type_mask = 1,
 442                 .tunnel_id_mask = 0xFFFFFFFF,
 443         },
 444         .drop_queue = 127,
 445 };
 446
 447 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 448
 449 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 450 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 451
 452 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 453 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 454
 455 uint16_t nb_tx_queue_stats_mappings = 0;
 456 uint16_t nb_rx_queue_stats_mappings = 0;
 457
 458 /*
 459  * Display zero values by default for xstats
 460  */
 461 uint8_t xstats_hide_zero;
 462
 463 unsigned int num_sockets = 0;
 464 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 465
 466 #ifdef RTE_LIBRTE_BITRATE
 467 /* Bitrate statistics */
 468 struct rte_stats_bitrates *bitrate_data;
 469 lcoreid_t bitrate_lcore_id;
 470 uint8_t bitrate_enabled;
 471 #endif
 472
 473 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 474 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 475
 476 struct vxlan_encap_conf vxlan_encap_conf = {
 477         .select_ipv4 = 1,
 478         .select_vlan = 0,
 479         .select_tos_ttl = 0,
 480         .vni = "\x00\x00\x00",
 481         .udp_src = 0,
 482         .udp_dst = RTE_BE16(4789),
 483         .ipv4_src = RTE_IPV4(127, 0, 0, 1),
 484         .ipv4_dst = RTE_IPV4(255, 255, 255, 255),
 485         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 486                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 487         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 488                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 489         .vlan_tci = 0,
 490         .ip_tos = 0,
 491         .ip_ttl = 255,
 492         .eth_src = "\x00\x00\x00\x00\x00\x00",
 493         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 494 };
 495
 496 struct nvgre_encap_conf nvgre_encap_conf = {
 497         .select_ipv4 = 1,
 498         .select_vlan = 0,
 499         .tni = "\x00\x00\x00",
 500         .ipv4_src = RTE_IPV4(127, 0, 0, 1),
 501         .ipv4_dst = RTE_IPV4(255, 255, 255, 255),
 502         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 503                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 504         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 505                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 506         .vlan_tci = 0,
 507         .eth_src = "\x00\x00\x00\x00\x00\x00",
 508         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 509 };
 510
 511 /* Forward function declarations */
 512 static void setup_attached_port(portid_t pi);
 513 static void map_port_queue_stats_mapping_registers(portid_t pi,
 514                                                    struct rte_port *port);
 515 static void check_all_ports_link_status(uint32_t port_mask);
 516 static int eth_event_callback(portid_t port_id,
 517                               enum rte_eth_event_type type,
 518                               void *param, void *ret_param);
 519 static void dev_event_callback(const char *device_name,
 520                                 enum rte_dev_event_type type,
 521                                 void *param);
 522
 523 /*
 524  * Check if all the ports are started.
 525  * If yes, return positive value. If not, return zero.
 526  */
 527 static int all_ports_started(void);
 528
 529 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 530 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 531
 532 /*
 533  * Helper function to check if socket is already discovered.
 534  * If yes, return positive value. If not, return zero.
 535  */
 536 int
 537 new_socket_id(unsigned int socket_id)
 538 {
 539         unsigned int i;
 540
 541         for (i = 0; i < num_sockets; i++) {
 542                 if (socket_ids[i] == socket_id)
 543                         return 0;
 544         }
 545         return 1;
 546 }
 547
 548 /*
 549  * Setup default configuration.
 550  */
 551 static void
 552 set_default_fwd_lcores_config(void)
 553 {
 554         unsigned int i;
 555         unsigned int nb_lc;
 556         unsigned int sock_num;
 557
 558         nb_lc = 0;
 559         for (i = 0; i < RTE_MAX_LCORE; i++) {
 560                 if (!rte_lcore_is_enabled(i))
 561                         continue;
 562                 sock_num = rte_lcore_to_socket_id(i);
 563                 if (new_socket_id(sock_num)) {
 564                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 565                                 rte_exit(EXIT_FAILURE,
 566                                          "Total sockets greater than %u\n",
 567                                          RTE_MAX_NUMA_NODES);
 568                         }
 569                         socket_ids[num_sockets++] = sock_num;
 570                 }
 571                 if (i == rte_get_master_lcore())
 572                         continue;
 573                 fwd_lcores_cpuids[nb_lc++] = i;
 574         }
 575         nb_lcores = (lcoreid_t) nb_lc;
 576         nb_cfg_lcores = nb_lcores;
 577         nb_fwd_lcores = 1;
 578 }
 579
 580 static void
 581 set_def_peer_eth_addrs(void)
 582 {
 583         portid_t i;
 584
 585         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 586                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 587                 peer_eth_addrs[i].addr_bytes[5] = i;
 588         }
 589 }
 590
 591 static void
 592 set_default_fwd_ports_config(void)
 593 {
 594         portid_t pt_id;
 595         int i = 0;
 596
 597         RTE_ETH_FOREACH_DEV(pt_id) {
 598                 fwd_ports_ids[i++] = pt_id;
 599
 600                 /* Update sockets info according to the attached device */
 601                 int socket_id = rte_eth_dev_socket_id(pt_id);
 602                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 603                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 604                                 rte_exit(EXIT_FAILURE,
 605                                          "Total sockets greater than %u\n",
 606                                          RTE_MAX_NUMA_NODES);
 607                         }
 608                         socket_ids[num_sockets++] = socket_id;
 609                 }
 610         }
 611
 612         nb_cfg_ports = nb_ports;
 613         nb_fwd_ports = nb_ports;
 614 }
 615
 616 void
 617 set_def_fwd_config(void)
 618 {
 619         set_default_fwd_lcores_config();
 620         set_def_peer_eth_addrs();
 621         set_default_fwd_ports_config();
 622 }
 623
 624 /* extremely pessimistic estimation of memory required to create a mempool */
 625 static int
 626 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 627 {
 628         unsigned int n_pages, mbuf_per_pg, leftover;
 629         uint64_t total_mem, mbuf_mem, obj_sz;
 630
 631         /* there is no good way to predict how much space the mempool will
 632          * occupy because it will allocate chunks on the fly, and some of those
 633          * will come from default DPDK memory while some will come from our
 634          * external memory, so just assume 128MB will be enough for everyone.
 635          */
 636         uint64_t hdr_mem = 128 << 20;
 637
 638         /* account for possible non-contiguousness */
 639         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 640         if (obj_sz > pgsz) {
 641                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 642                 return -1;
 643         }
 644
 645         mbuf_per_pg = pgsz / obj_sz;
 646         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 647         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 648
 649         mbuf_mem = n_pages * pgsz;
 650
 651         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 652
 653         if (total_mem > SIZE_MAX) {
 654                 TESTPMD_LOG(ERR, "Memory size too big\n");
 655                 return -1;
 656         }
 657         *out = (size_t)total_mem;
 658
 659         return 0;
 660 }
 661
 662 static int
 663 pagesz_flags(uint64_t page_sz)
 664 {
 665         /* as per mmap() manpage, all page sizes are log2 of page size
 666          * shifted by MAP_HUGE_SHIFT
 667          */
 668         int log2 = rte_log2_u64(page_sz);
 669
 670         return (log2 << HUGE_SHIFT);
 671 }
 672
 673 static void *
 674 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 675 {
 676         void *addr;
 677         int flags;
 678
 679         /* allocate anonymous hugepages */
 680         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 681         if (huge)
 682                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 683
 684         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 685         if (addr == MAP_FAILED)
 686                 return NULL;
 687
 688         return addr;
 689 }
 690
 691 struct extmem_param {
 692         void *addr;
 693         size_t len;
 694         size_t pgsz;
 695         rte_iova_t *iova_table;
 696         unsigned int iova_table_len;
 697 };
 698
 699 static int
 700 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 701                 bool huge)
 702 {
 703         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 704                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 705         unsigned int cur_page, n_pages, pgsz_idx;
 706         size_t mem_sz, cur_pgsz;
 707         rte_iova_t *iovas = NULL;
 708         void *addr;
 709         int ret;
 710
 711         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 712                 /* skip anything that is too big */
 713                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 714                         continue;
 715
 716                 cur_pgsz = pgsizes[pgsz_idx];
 717
 718                 /* if we were told not to allocate hugepages, override */
 719                 if (!huge)
 720                         cur_pgsz = sysconf(_SC_PAGESIZE);
 721
 722                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 723                 if (ret < 0) {
 724                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 725                         return -1;
 726                 }
 727
 728                 /* allocate our memory */
 729                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 730
 731                 /* if we couldn't allocate memory with a specified page size,
 732                  * that doesn't mean we can't do it with other page sizes, so
 733                  * try another one.
 734                  */
 735                 if (addr == NULL)
 736                         continue;
 737
 738                 /* store IOVA addresses for every page in this memory area */
 739                 n_pages = mem_sz / cur_pgsz;
 740
 741                 iovas = malloc(sizeof(*iovas) * n_pages);
 742
 743                 if (iovas == NULL) {
 744                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 745                         goto fail;
 746                 }
 747                 /* lock memory if it's not huge pages */
 748                 if (!huge)
 749                         mlock(addr, mem_sz);
 750
 751                 /* populate IOVA addresses */
 752                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 753                         rte_iova_t iova;
 754                         size_t offset;
 755                         void *cur;
 756
 757                         offset = cur_pgsz * cur_page;
 758                         cur = RTE_PTR_ADD(addr, offset);
 759
 760                         /* touch the page before getting its IOVA */
 761                         *(volatile char *)cur = 0;
 762
 763                         iova = rte_mem_virt2iova(cur);
 764
 765                         iovas[cur_page] = iova;
 766                 }
 767
 768                 break;
 769         }
 770         /* if we couldn't allocate anything */
 771         if (iovas == NULL)
 772                 return -1;
 773
 774         param->addr = addr;
 775         param->len = mem_sz;
 776         param->pgsz = cur_pgsz;
 777         param->iova_table = iovas;
 778         param->iova_table_len = n_pages;
 779
 780         return 0;
 781 fail:
 782         if (iovas)
 783                 free(iovas);
 784         if (addr)
 785                 munmap(addr, mem_sz);
 786
 787         return -1;
 788 }
 789
 790 static int
 791 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 792 {
 793         struct extmem_param param;
 794         int socket_id, ret;
 795
 796         memset(&param, 0, sizeof(param));
 797
 798         /* check if our heap exists */
 799         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 800         if (socket_id < 0) {
 801                 /* create our heap */
 802                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 803                 if (ret < 0) {
 804                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 805                         return -1;
 806                 }
 807         }
 808
 809         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 810         if (ret < 0) {
 811                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 812                 return -1;
 813         }
 814
 815         /* we now have a valid memory area, so add it to heap */
 816         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 817                         param.addr, param.len, param.iova_table,
 818                         param.iova_table_len, param.pgsz);
 819
 820         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 821
 822         /* not needed any more */
 823         free(param.iova_table);
 824
 825         if (ret < 0) {
 826                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 827                 munmap(param.addr, param.len);
 828                 return -1;
 829         }
 830
 831         /* success */
 832
 833         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 834                         param.len >> 20);
 835
 836         return 0;
 837 }
 838 static void
 839 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 840              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 841 {
 842         uint16_t pid = 0;
 843         int ret;
 844
 845         RTE_ETH_FOREACH_DEV(pid) {
 846                 struct rte_eth_dev *dev =
 847                         &rte_eth_devices[pid];
 848
 849                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 850                                         memhdr->len);
 851                 if (ret) {
 852                         TESTPMD_LOG(DEBUG,
 853                                     "unable to DMA unmap addr 0x%p "
 854                                     "for device %s\n",
 855                                     memhdr->addr, dev->data->name);
 856                 }
 857         }
 858         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 859         if (ret) {
 860                 TESTPMD_LOG(DEBUG,
 861                             "unable to un-register addr 0x%p\n", memhdr->addr);
 862         }
 863 }
 864
 865 static void
 866 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 867            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 868 {
 869         uint16_t pid = 0;
 870         size_t page_size = sysconf(_SC_PAGESIZE);
 871         int ret;
 872
 873         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 874                                   page_size);
 875         if (ret) {
 876                 TESTPMD_LOG(DEBUG,
 877                             "unable to register addr 0x%p\n", memhdr->addr);
 878                 return;
 879         }
 880         RTE_ETH_FOREACH_DEV(pid) {
 881                 struct rte_eth_dev *dev =
 882                         &rte_eth_devices[pid];
 883
 884                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 885                                       memhdr->len);
 886                 if (ret) {
 887                         TESTPMD_LOG(DEBUG,
 888                                     "unable to DMA map addr 0x%p "
 889                                     "for device %s\n",
 890                                     memhdr->addr, dev->data->name);
 891                 }
 892         }
 893 }
 894
 895 /*
 896  * Configuration initialisation done once at init time.
 897  */
 898 static struct rte_mempool *
 899 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 900                  unsigned int socket_id)
 901 {
 902         char pool_name[RTE_MEMPOOL_NAMESIZE];
 903         struct rte_mempool *rte_mp = NULL;
 904         uint32_t mb_size;
 905
 906         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 907         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 908
 909         TESTPMD_LOG(INFO,
 910                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 911                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 912
 913         switch (mp_alloc_type) {
 914         case MP_ALLOC_NATIVE:
 915                 {
 916                         /* wrapper to rte_mempool_create() */
 917                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 918                                         rte_mbuf_best_mempool_ops());
 919                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 920                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 921                         break;
 922                 }
 923         case MP_ALLOC_ANON:
 924                 {
 925                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 926                                 mb_size, (unsigned int) mb_mempool_cache,
 927                                 sizeof(struct rte_pktmbuf_pool_private),
 928                                 socket_id, mempool_flags);
 929                         if (rte_mp == NULL)
 930                                 goto err;
 931
 932                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 933                                 rte_mempool_free(rte_mp);
 934                                 rte_mp = NULL;
 935                                 goto err;
 936                         }
 937                         rte_pktmbuf_pool_init(rte_mp, NULL);
 938                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 939                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 940                         break;
 941                 }
 942         case MP_ALLOC_XMEM:
 943         case MP_ALLOC_XMEM_HUGE:
 944                 {
 945                         int heap_socket;
 946                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 947
 948                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 949                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 950
 951                         heap_socket =
 952                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 953                         if (heap_socket < 0)
 954                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 955
 956                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 957                                         rte_mbuf_best_mempool_ops());
 958                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 959                                         mb_mempool_cache, 0, mbuf_seg_size,
 960                                         heap_socket);
 961                         break;
 962                 }
 963         default:
 964                 {
 965                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 966                 }
 967         }
 968
 969 err:
 970         if (rte_mp == NULL) {
 971                 rte_exit(EXIT_FAILURE,
 972                         "Creation of mbuf pool for socket %u failed: %s\n",
 973                         socket_id, rte_strerror(rte_errno));
 974         } else if (verbose_level > 0) {
 975                 rte_mempool_dump(stdout, rte_mp);
 976         }
 977         return rte_mp;
 978 }
 979
 980 /*
 981  * Check given socket id is valid or not with NUMA mode,
 982  * if valid, return 0, else return -1
 983  */
 984 static int
 985 check_socket_id(const unsigned int socket_id)
 986 {
 987         static int warning_once = 0;
 988
 989         if (new_socket_id(socket_id)) {
 990                 if (!warning_once && numa_support)
 991                         printf("Warning: NUMA should be configured manually by"
 992                                " using --port-numa-config and"
 993                                " --ring-numa-config parameters along with"
 994                                " --numa.\n");
 995                 warning_once = 1;
 996                 return -1;
 997         }
 998         return 0;
 999 }
1000
1001 /*
1002  * Get the allowed maximum number of RX queues.
1003  * *pid return the port id which has minimal value of
1004  * max_rx_queues in all ports.
1005  */
1006 queueid_t
1007 get_allowed_max_nb_rxq(portid_t *pid)
1008 {
1009         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
1010         portid_t pi;
1011         struct rte_eth_dev_info dev_info;
1012
1013         RTE_ETH_FOREACH_DEV(pi) {
1014                 rte_eth_dev_info_get(pi, &dev_info);
1015                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1016                         allowed_max_rxq = dev_info.max_rx_queues;
1017                         *pid = pi;
1018                 }
1019         }
1020         return allowed_max_rxq;
1021 }
1022
1023 /*
1024  * Check input rxq is valid or not.
1025  * If input rxq is not greater than any of maximum number
1026  * of RX queues of all ports, it is valid.
1027  * if valid, return 0, else return -1
1028  */
1029 int
1030 check_nb_rxq(queueid_t rxq)
1031 {
1032         queueid_t allowed_max_rxq;
1033         portid_t pid = 0;
1034
1035         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1036         if (rxq > allowed_max_rxq) {
1037                 printf("Fail: input rxq (%u) can't be greater "
1038                        "than max_rx_queues (%u) of port %u\n",
1039                        rxq,
1040                        allowed_max_rxq,
1041                        pid);
1042                 return -1;
1043         }
1044         return 0;
1045 }
1046
1047 /*
1048  * Get the allowed maximum number of TX queues.
1049  * *pid return the port id which has minimal value of
1050  * max_tx_queues in all ports.
1051  */
1052 queueid_t
1053 get_allowed_max_nb_txq(portid_t *pid)
1054 {
1055         queueid_t allowed_max_txq = MAX_QUEUE_ID;
1056         portid_t pi;
1057         struct rte_eth_dev_info dev_info;
1058
1059         RTE_ETH_FOREACH_DEV(pi) {
1060                 rte_eth_dev_info_get(pi, &dev_info);
1061                 if (dev_info.max_tx_queues < allowed_max_txq) {
1062                         allowed_max_txq = dev_info.max_tx_queues;
1063                         *pid = pi;
1064                 }
1065         }
1066         return allowed_max_txq;
1067 }
1068
1069 /*
1070  * Check input txq is valid or not.
1071  * If input txq is not greater than any of maximum number
1072  * of TX queues of all ports, it is valid.
1073  * if valid, return 0, else return -1
1074  */
1075 int
1076 check_nb_txq(queueid_t txq)
1077 {
1078         queueid_t allowed_max_txq;
1079         portid_t pid = 0;
1080
1081         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1082         if (txq > allowed_max_txq) {
1083                 printf("Fail: input txq (%u) can't be greater "
1084                        "than max_tx_queues (%u) of port %u\n",
1085                        txq,
1086                        allowed_max_txq,
1087                        pid);
1088                 return -1;
1089         }
1090         return 0;
1091 }
1092
1093 static void
1094 init_config(void)
1095 {
1096         portid_t pid;
1097         struct rte_port *port;
1098         struct rte_mempool *mbp;
1099         unsigned int nb_mbuf_per_pool;
1100         lcoreid_t  lc_id;
1101         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1102         struct rte_gro_param gro_param;
1103         uint32_t gso_types;
1104         uint16_t data_size;
1105         bool warning = 0;
1106         int k;
1107
1108         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1109
1110         /* Configuration of logical cores. */
1111         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1112                                 sizeof(struct fwd_lcore *) * nb_lcores,
1113                                 RTE_CACHE_LINE_SIZE);
1114         if (fwd_lcores == NULL) {
1115                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1116                                                         "failed\n", nb_lcores);
1117         }
1118         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1119                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1120                                                sizeof(struct fwd_lcore),
1121                                                RTE_CACHE_LINE_SIZE);
1122                 if (fwd_lcores[lc_id] == NULL) {
1123                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1124                                                                 "failed\n");
1125                 }
1126                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1127         }
1128
1129         RTE_ETH_FOREACH_DEV(pid) {
1130                 port = &ports[pid];
1131                 /* Apply default TxRx configuration for all ports */
1132                 port->dev_conf.txmode = tx_mode;
1133                 port->dev_conf.rxmode = rx_mode;
1134                 rte_eth_dev_info_get(pid, &port->dev_info);
1135
1136                 if (!(port->dev_info.tx_offload_capa &
1137                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1138                         port->dev_conf.txmode.offloads &=
1139                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1140                 if (!(port->dev_info.tx_offload_capa &
1141                         DEV_TX_OFFLOAD_MATCH_METADATA))
1142                         port->dev_conf.txmode.offloads &=
1143                                 ~DEV_TX_OFFLOAD_MATCH_METADATA;
1144                 if (numa_support) {
1145                         if (port_numa[pid] != NUMA_NO_CONFIG)
1146                                 port_per_socket[port_numa[pid]]++;
1147                         else {
1148                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1149
1150                                 /*
1151                                  * if socket_id is invalid,
1152                                  * set to the first available socket.
1153                                  */
1154                                 if (check_socket_id(socket_id) < 0)
1155                                         socket_id = socket_ids[0];
1156                                 port_per_socket[socket_id]++;
1157                         }
1158                 }
1159
1160                 /* Apply Rx offloads configuration */
1161                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1162                         port->rx_conf[k].offloads =
1163                                 port->dev_conf.rxmode.offloads;
1164                 /* Apply Tx offloads configuration */
1165                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1166                         port->tx_conf[k].offloads =
1167                                 port->dev_conf.txmode.offloads;
1168
1169                 /* set flag to initialize port/queue */
1170                 port->need_reconfig = 1;
1171                 port->need_reconfig_queues = 1;
1172                 port->tx_metadata = 0;
1173
1174                 /* Check for maximum number of segments per MTU. Accordingly
1175                  * update the mbuf data size.
1176                  */
1177                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX) {
1178                         data_size = rx_mode.max_rx_pkt_len /
1179                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1180
1181                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1182                                                         mbuf_data_size) {
1183                                 mbuf_data_size = data_size +
1184                                                  RTE_PKTMBUF_HEADROOM;
1185                                 warning = 1;
1186                         }
1187                 }
1188         }
1189
1190         if (warning)
1191                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1192                             mbuf_data_size);
1193
1194         /*
1195          * Create pools of mbuf.
1196          * If NUMA support is disabled, create a single pool of mbuf in
1197          * socket 0 memory by default.
1198          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1199          *
1200          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1201          * nb_txd can be configured at run time.
1202          */
1203         if (param_total_num_mbufs)
1204                 nb_mbuf_per_pool = param_total_num_mbufs;
1205         else {
1206                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1207                         (nb_lcores * mb_mempool_cache) +
1208                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1209                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1210         }
1211
1212         if (numa_support) {
1213                 uint8_t i;
1214
1215                 for (i = 0; i < num_sockets; i++)
1216                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1217                                                        nb_mbuf_per_pool,
1218                                                        socket_ids[i]);
1219         } else {
1220                 if (socket_num == UMA_NO_CONFIG)
1221                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1222                                                        nb_mbuf_per_pool, 0);
1223                 else
1224                         mempools[socket_num] = mbuf_pool_create
1225                                                         (mbuf_data_size,
1226                                                          nb_mbuf_per_pool,
1227                                                          socket_num);
1228         }
1229
1230         init_port_config();
1231
1232         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1233                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1234         /*
1235          * Records which Mbuf pool to use by each logical core, if needed.
1236          */
1237         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1238                 mbp = mbuf_pool_find(
1239                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1240
1241                 if (mbp == NULL)
1242                         mbp = mbuf_pool_find(0);
1243                 fwd_lcores[lc_id]->mbp = mbp;
1244                 /* initialize GSO context */
1245                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1246                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1247                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1248                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1249                         RTE_ETHER_CRC_LEN;
1250                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1251         }
1252
1253         /* Configuration of packet forwarding streams. */
1254         if (init_fwd_streams() < 0)
1255                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1256
1257         fwd_config_setup();
1258
1259         /* create a gro context for each lcore */
1260         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1261         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1262         gro_param.max_item_per_flow = MAX_PKT_BURST;
1263         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1264                 gro_param.socket_id = rte_lcore_to_socket_id(
1265                                 fwd_lcores_cpuids[lc_id]);
1266                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1267                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1268                         rte_exit(EXIT_FAILURE,
1269                                         "rte_gro_ctx_create() failed\n");
1270                 }
1271         }
1272
1273 #if defined RTE_LIBRTE_PMD_SOFTNIC
1274         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1275                 RTE_ETH_FOREACH_DEV(pid) {
1276                         port = &ports[pid];
1277                         const char *driver = port->dev_info.driver_name;
1278
1279                         if (strcmp(driver, "net_softnic") == 0)
1280                                 port->softport.fwd_lcore_arg = fwd_lcores;
1281                 }
1282         }
1283 #endif
1284
1285 }
1286
1287
1288 void
1289 reconfig(portid_t new_port_id, unsigned socket_id)
1290 {
1291         struct rte_port *port;
1292
1293         /* Reconfiguration of Ethernet ports. */
1294         port = &ports[new_port_id];
1295         rte_eth_dev_info_get(new_port_id, &port->dev_info);
1296
1297         /* set flag to initialize port/queue */
1298         port->need_reconfig = 1;
1299         port->need_reconfig_queues = 1;
1300         port->socket_id = socket_id;
1301
1302         init_port_config();
1303 }
1304
1305
1306 int
1307 init_fwd_streams(void)
1308 {
1309         portid_t pid;
1310         struct rte_port *port;
1311         streamid_t sm_id, nb_fwd_streams_new;
1312         queueid_t q;
1313
1314         /* set socket id according to numa or not */
1315         RTE_ETH_FOREACH_DEV(pid) {
1316                 port = &ports[pid];
1317                 if (nb_rxq > port->dev_info.max_rx_queues) {
1318                         printf("Fail: nb_rxq(%d) is greater than "
1319                                 "max_rx_queues(%d)\n", nb_rxq,
1320                                 port->dev_info.max_rx_queues);
1321                         return -1;
1322                 }
1323                 if (nb_txq > port->dev_info.max_tx_queues) {
1324                         printf("Fail: nb_txq(%d) is greater than "
1325                                 "max_tx_queues(%d)\n", nb_txq,
1326                                 port->dev_info.max_tx_queues);
1327                         return -1;
1328                 }
1329                 if (numa_support) {
1330                         if (port_numa[pid] != NUMA_NO_CONFIG)
1331                                 port->socket_id = port_numa[pid];
1332                         else {
1333                                 port->socket_id = rte_eth_dev_socket_id(pid);
1334
1335                                 /*
1336                                  * if socket_id is invalid,
1337                                  * set to the first available socket.
1338                                  */
1339                                 if (check_socket_id(port->socket_id) < 0)
1340                                         port->socket_id = socket_ids[0];
1341                         }
1342                 }
1343                 else {
1344                         if (socket_num == UMA_NO_CONFIG)
1345                                 port->socket_id = 0;
1346                         else
1347                                 port->socket_id = socket_num;
1348                 }
1349         }
1350
1351         q = RTE_MAX(nb_rxq, nb_txq);
1352         if (q == 0) {
1353                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1354                 return -1;
1355         }
1356         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1357         if (nb_fwd_streams_new == nb_fwd_streams)
1358                 return 0;
1359         /* clear the old */
1360         if (fwd_streams != NULL) {
1361                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1362                         if (fwd_streams[sm_id] == NULL)
1363                                 continue;
1364                         rte_free(fwd_streams[sm_id]);
1365                         fwd_streams[sm_id] = NULL;
1366                 }
1367                 rte_free(fwd_streams);
1368                 fwd_streams = NULL;
1369         }
1370
1371         /* init new */
1372         nb_fwd_streams = nb_fwd_streams_new;
1373         if (nb_fwd_streams) {
1374                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1375                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1376                         RTE_CACHE_LINE_SIZE);
1377                 if (fwd_streams == NULL)
1378                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1379                                  " (struct fwd_stream *)) failed\n",
1380                                  nb_fwd_streams);
1381
1382                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1383                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1384                                 " struct fwd_stream", sizeof(struct fwd_stream),
1385                                 RTE_CACHE_LINE_SIZE);
1386                         if (fwd_streams[sm_id] == NULL)
1387                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1388                                          "(struct fwd_stream) failed\n");
1389                 }
1390         }
1391
1392         return 0;
1393 }
1394
1395 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1396 static void
1397 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1398 {
1399         unsigned int total_burst;
1400         unsigned int nb_burst;
1401         unsigned int burst_stats[3];
1402         uint16_t pktnb_stats[3];
1403         uint16_t nb_pkt;
1404         int burst_percent[3];
1405
1406         /*
1407          * First compute the total number of packet bursts and the
1408          * two highest numbers of bursts of the same number of packets.
1409          */
1410         total_burst = 0;
1411         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1412         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1413         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1414                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1415                 if (nb_burst == 0)
1416                         continue;
1417                 total_burst += nb_burst;
1418                 if (nb_burst > burst_stats[0]) {
1419                         burst_stats[1] = burst_stats[0];
1420                         pktnb_stats[1] = pktnb_stats[0];
1421                         burst_stats[0] = nb_burst;
1422                         pktnb_stats[0] = nb_pkt;
1423                 } else if (nb_burst > burst_stats[1]) {
1424                         burst_stats[1] = nb_burst;
1425                         pktnb_stats[1] = nb_pkt;
1426                 }
1427         }
1428         if (total_burst == 0)
1429                 return;
1430         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1431         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1432                burst_percent[0], (int) pktnb_stats[0]);
1433         if (burst_stats[0] == total_burst) {
1434                 printf("]\n");
1435                 return;
1436         }
1437         if (burst_stats[0] + burst_stats[1] == total_burst) {
1438                 printf(" + %d%% of %d pkts]\n",
1439                        100 - burst_percent[0], pktnb_stats[1]);
1440                 return;
1441         }
1442         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1443         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1444         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1445                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1446                 return;
1447         }
1448         printf(" + %d%% of %d pkts + %d%% of others]\n",
1449                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1450 }
1451 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1452
1453 static void
1454 fwd_stream_stats_display(streamid_t stream_id)
1455 {
1456         struct fwd_stream *fs;
1457         static const char *fwd_top_stats_border = "-------";
1458
1459         fs = fwd_streams[stream_id];
1460         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1461             (fs->fwd_dropped == 0))
1462                 return;
1463         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1464                "TX Port=%2d/Queue=%2d %s\n",
1465                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1466                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1467         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1468                " TX-dropped: %-14"PRIu64,
1469                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1470
1471         /* if checksum mode */
1472         if (cur_fwd_eng == &csum_fwd_engine) {
1473                 printf("  RX- bad IP checksum: %-14"PRIu64
1474                        "  Rx- bad L4 checksum: %-14"PRIu64
1475                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1476                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1477                         fs->rx_bad_outer_l4_csum);
1478         } else {
1479                 printf("\n");
1480         }
1481
1482 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1483         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1484         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1485 #endif
1486 }
1487
1488 void
1489 fwd_stats_display(void)
1490 {
1491         static const char *fwd_stats_border = "----------------------";
1492         static const char *acc_stats_border = "+++++++++++++++";
1493         struct {
1494                 struct fwd_stream *rx_stream;
1495                 struct fwd_stream *tx_stream;
1496                 uint64_t tx_dropped;
1497                 uint64_t rx_bad_ip_csum;
1498                 uint64_t rx_bad_l4_csum;
1499                 uint64_t rx_bad_outer_l4_csum;
1500         } ports_stats[RTE_MAX_ETHPORTS];
1501         uint64_t total_rx_dropped = 0;
1502         uint64_t total_tx_dropped = 0;
1503         uint64_t total_rx_nombuf = 0;
1504         struct rte_eth_stats stats;
1505 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1506         uint64_t fwd_cycles = 0;
1507 #endif
1508         uint64_t total_recv = 0;
1509         uint64_t total_xmit = 0;
1510         struct rte_port *port;
1511         streamid_t sm_id;
1512         portid_t pt_id;
1513         int i;
1514
1515         memset(ports_stats, 0, sizeof(ports_stats));
1516
1517         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1518                 struct fwd_stream *fs = fwd_streams[sm_id];
1519
1520                 if (cur_fwd_config.nb_fwd_streams >
1521                     cur_fwd_config.nb_fwd_ports) {
1522                         fwd_stream_stats_display(sm_id);
1523                 } else {
1524                         ports_stats[fs->tx_port].tx_stream = fs;
1525                         ports_stats[fs->rx_port].rx_stream = fs;
1526                 }
1527
1528                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1529
1530                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1531                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1532                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1533                                 fs->rx_bad_outer_l4_csum;
1534
1535 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1536                 fwd_cycles += fs->core_cycles;
1537 #endif
1538         }
1539         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1540                 uint8_t j;
1541
1542                 pt_id = fwd_ports_ids[i];
1543                 port = &ports[pt_id];
1544
1545                 rte_eth_stats_get(pt_id, &stats);
1546                 stats.ipackets -= port->stats.ipackets;
1547                 stats.opackets -= port->stats.opackets;
1548                 stats.ibytes -= port->stats.ibytes;
1549                 stats.obytes -= port->stats.obytes;
1550                 stats.imissed -= port->stats.imissed;
1551                 stats.oerrors -= port->stats.oerrors;
1552                 stats.rx_nombuf -= port->stats.rx_nombuf;
1553
1554                 total_recv += stats.ipackets;
1555                 total_xmit += stats.opackets;
1556                 total_rx_dropped += stats.imissed;
1557                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1558                 total_tx_dropped += stats.oerrors;
1559                 total_rx_nombuf  += stats.rx_nombuf;
1560
1561                 printf("\n  %s Forward statistics for port %-2d %s\n",
1562                        fwd_stats_border, pt_id, fwd_stats_border);
1563
1564                 if (!port->rx_queue_stats_mapping_enabled &&
1565                     !port->tx_queue_stats_mapping_enabled) {
1566                         printf("  RX-packets: %-14"PRIu64
1567                                " RX-dropped: %-14"PRIu64
1568                                "RX-total: %-"PRIu64"\n",
1569                                stats.ipackets, stats.imissed,
1570                                stats.ipackets + stats.imissed);
1571
1572                         if (cur_fwd_eng == &csum_fwd_engine)
1573                                 printf("  Bad-ipcsum: %-14"PRIu64
1574                                        " Bad-l4csum: %-14"PRIu64
1575                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1576                                        ports_stats[pt_id].rx_bad_ip_csum,
1577                                        ports_stats[pt_id].rx_bad_l4_csum,
1578                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1579                         if (stats.ierrors + stats.rx_nombuf > 0) {
1580                                 printf("  RX-error: %-"PRIu64"\n",
1581                                        stats.ierrors);
1582                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1583                                        stats.rx_nombuf);
1584                         }
1585
1586                         printf("  TX-packets: %-14"PRIu64
1587                                " TX-dropped: %-14"PRIu64
1588                                "TX-total: %-"PRIu64"\n",
1589                                stats.opackets, ports_stats[pt_id].tx_dropped,
1590                                stats.opackets + ports_stats[pt_id].tx_dropped);
1591                 } else {
1592                         printf("  RX-packets:             %14"PRIu64
1593                                "    RX-dropped:%14"PRIu64
1594                                "    RX-total:%14"PRIu64"\n",
1595                                stats.ipackets, stats.imissed,
1596                                stats.ipackets + stats.imissed);
1597
1598                         if (cur_fwd_eng == &csum_fwd_engine)
1599                                 printf("  Bad-ipcsum:%14"PRIu64
1600                                        "    Bad-l4csum:%14"PRIu64
1601                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1602                                        ports_stats[pt_id].rx_bad_ip_csum,
1603                                        ports_stats[pt_id].rx_bad_l4_csum,
1604                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1605                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1606                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1607                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1608                                        stats.rx_nombuf);
1609                         }
1610
1611                         printf("  TX-packets:             %14"PRIu64
1612                                "    TX-dropped:%14"PRIu64
1613                                "    TX-total:%14"PRIu64"\n",
1614                                stats.opackets, ports_stats[pt_id].tx_dropped,
1615                                stats.opackets + ports_stats[pt_id].tx_dropped);
1616                 }
1617
1618 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1619                 if (ports_stats[pt_id].rx_stream)
1620                         pkt_burst_stats_display("RX",
1621                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1622                 if (ports_stats[pt_id].tx_stream)
1623                         pkt_burst_stats_display("TX",
1624                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1625 #endif
1626
1627                 if (port->rx_queue_stats_mapping_enabled) {
1628                         printf("\n");
1629                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1630                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1631                                        "     RX-errors:%14"PRIu64
1632                                        "    RX-bytes:%14"PRIu64"\n",
1633                                        j, stats.q_ipackets[j],
1634                                        stats.q_errors[j], stats.q_ibytes[j]);
1635                         }
1636                         printf("\n");
1637                 }
1638                 if (port->tx_queue_stats_mapping_enabled) {
1639                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1640                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1641                                        "                                 TX-bytes:%14"
1642                                        PRIu64"\n",
1643                                        j, stats.q_opackets[j],
1644                                        stats.q_obytes[j]);
1645                         }
1646                 }
1647
1648                 printf("  %s--------------------------------%s\n",
1649                        fwd_stats_border, fwd_stats_border);
1650         }
1651
1652         printf("\n  %s Accumulated forward statistics for all ports"
1653                "%s\n",
1654                acc_stats_border, acc_stats_border);
1655         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1656                "%-"PRIu64"\n"
1657                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1658                "%-"PRIu64"\n",
1659                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1660                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1661         if (total_rx_nombuf > 0)
1662                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1663         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1664                "%s\n",
1665                acc_stats_border, acc_stats_border);
1666 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1667         if (total_recv > 0)
1668                 printf("\n  CPU cycles/packet=%u (total cycles="
1669                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1670                        (unsigned int)(fwd_cycles / total_recv),
1671                        fwd_cycles, total_recv);
1672 #endif
1673 }
1674
1675 void
1676 fwd_stats_reset(void)
1677 {
1678         streamid_t sm_id;
1679         portid_t pt_id;
1680         int i;
1681
1682         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1683                 pt_id = fwd_ports_ids[i];
1684                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1685         }
1686         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1687                 struct fwd_stream *fs = fwd_streams[sm_id];
1688
1689                 fs->rx_packets = 0;
1690                 fs->tx_packets = 0;
1691                 fs->fwd_dropped = 0;
1692                 fs->rx_bad_ip_csum = 0;
1693                 fs->rx_bad_l4_csum = 0;
1694                 fs->rx_bad_outer_l4_csum = 0;
1695
1696 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1697                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1698                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1699 #endif
1700 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1701                 fs->core_cycles = 0;
1702 #endif
1703         }
1704 }
1705
1706 static void
1707 flush_fwd_rx_queues(void)
1708 {
1709         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1710         portid_t  rxp;
1711         portid_t port_id;
1712         queueid_t rxq;
1713         uint16_t  nb_rx;
1714         uint16_t  i;
1715         uint8_t   j;
1716         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1717         uint64_t timer_period;
1718
1719         /* convert to number of cycles */
1720         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1721
1722         for (j = 0; j < 2; j++) {
1723                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1724                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1725                                 port_id = fwd_ports_ids[rxp];
1726                                 /**
1727                                 * testpmd can stuck in the below do while loop
1728                                 * if rte_eth_rx_burst() always returns nonzero
1729                                 * packets. So timer is added to exit this loop
1730                                 * after 1sec timer expiry.
1731                                 */
1732                                 prev_tsc = rte_rdtsc();
1733                                 do {
1734                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1735                                                 pkts_burst, MAX_PKT_BURST);
1736                                         for (i = 0; i < nb_rx; i++)
1737                                                 rte_pktmbuf_free(pkts_burst[i]);
1738
1739                                         cur_tsc = rte_rdtsc();
1740                                         diff_tsc = cur_tsc - prev_tsc;
1741                                         timer_tsc += diff_tsc;
1742                                 } while ((nb_rx > 0) &&
1743                                         (timer_tsc < timer_period));
1744                                 timer_tsc = 0;
1745                         }
1746                 }
1747                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1748         }
1749 }
1750
1751 static void
1752 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1753 {
1754         struct fwd_stream **fsm;
1755         streamid_t nb_fs;
1756         streamid_t sm_id;
1757 #ifdef RTE_LIBRTE_BITRATE
1758         uint64_t tics_per_1sec;
1759         uint64_t tics_datum;
1760         uint64_t tics_current;
1761         uint16_t i, cnt_ports;
1762
1763         cnt_ports = nb_ports;
1764         tics_datum = rte_rdtsc();
1765         tics_per_1sec = rte_get_timer_hz();
1766 #endif
1767         fsm = &fwd_streams[fc->stream_idx];
1768         nb_fs = fc->stream_nb;
1769         do {
1770                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1771                         (*pkt_fwd)(fsm[sm_id]);
1772 #ifdef RTE_LIBRTE_BITRATE
1773                 if (bitrate_enabled != 0 &&
1774                                 bitrate_lcore_id == rte_lcore_id()) {
1775                         tics_current = rte_rdtsc();
1776                         if (tics_current - tics_datum >= tics_per_1sec) {
1777                                 /* Periodic bitrate calculation */
1778                                 for (i = 0; i < cnt_ports; i++)
1779                                         rte_stats_bitrate_calc(bitrate_data,
1780                                                 ports_ids[i]);
1781                                 tics_datum = tics_current;
1782                         }
1783                 }
1784 #endif
1785 #ifdef RTE_LIBRTE_LATENCY_STATS
1786                 if (latencystats_enabled != 0 &&
1787                                 latencystats_lcore_id == rte_lcore_id())
1788                         rte_latencystats_update();
1789 #endif
1790
1791         } while (! fc->stopped);
1792 }
1793
1794 static int
1795 start_pkt_forward_on_core(void *fwd_arg)
1796 {
1797         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1798                              cur_fwd_config.fwd_eng->packet_fwd);
1799         return 0;
1800 }
1801
1802 /*
1803  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1804  * Used to start communication flows in network loopback test configurations.
1805  */
1806 static int
1807 run_one_txonly_burst_on_core(void *fwd_arg)
1808 {
1809         struct fwd_lcore *fwd_lc;
1810         struct fwd_lcore tmp_lcore;
1811
1812         fwd_lc = (struct fwd_lcore *) fwd_arg;
1813         tmp_lcore = *fwd_lc;
1814         tmp_lcore.stopped = 1;
1815         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1816         return 0;
1817 }
1818
1819 /*
1820  * Launch packet forwarding:
1821  *     - Setup per-port forwarding context.
1822  *     - launch logical cores with their forwarding configuration.
1823  */
1824 static void
1825 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1826 {
1827         port_fwd_begin_t port_fwd_begin;
1828         unsigned int i;
1829         unsigned int lc_id;
1830         int diag;
1831
1832         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1833         if (port_fwd_begin != NULL) {
1834                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1835                         (*port_fwd_begin)(fwd_ports_ids[i]);
1836         }
1837         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1838                 lc_id = fwd_lcores_cpuids[i];
1839                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1840                         fwd_lcores[i]->stopped = 0;
1841                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1842                                                      fwd_lcores[i], lc_id);
1843                         if (diag != 0)
1844                                 printf("launch lcore %u failed - diag=%d\n",
1845                                        lc_id, diag);
1846                 }
1847         }
1848 }
1849
1850 /*
1851  * Launch packet forwarding configuration.
1852  */
1853 void
1854 start_packet_forwarding(int with_tx_first)
1855 {
1856         port_fwd_begin_t port_fwd_begin;
1857         port_fwd_end_t  port_fwd_end;
1858         struct rte_port *port;
1859         unsigned int i;
1860         portid_t   pt_id;
1861
1862         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1863                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1864
1865         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1866                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1867
1868         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1869                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1870                 (!nb_rxq || !nb_txq))
1871                 rte_exit(EXIT_FAILURE,
1872                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1873                         cur_fwd_eng->fwd_mode_name);
1874
1875         if (all_ports_started() == 0) {
1876                 printf("Not all ports were started\n");
1877                 return;
1878         }
1879         if (test_done == 0) {
1880                 printf("Packet forwarding already started\n");
1881                 return;
1882         }
1883
1884
1885         if(dcb_test) {
1886                 for (i = 0; i < nb_fwd_ports; i++) {
1887                         pt_id = fwd_ports_ids[i];
1888                         port = &ports[pt_id];
1889                         if (!port->dcb_flag) {
1890                                 printf("In DCB mode, all forwarding ports must "
1891                                        "be configured in this mode.\n");
1892                                 return;
1893                         }
1894                 }
1895                 if (nb_fwd_lcores == 1) {
1896                         printf("In DCB mode,the nb forwarding cores "
1897                                "should be larger than 1.\n");
1898                         return;
1899                 }
1900         }
1901         test_done = 0;
1902
1903         fwd_config_setup();
1904
1905         if(!no_flush_rx)
1906                 flush_fwd_rx_queues();
1907
1908         pkt_fwd_config_display(&cur_fwd_config);
1909         rxtx_config_display();
1910
1911         fwd_stats_reset();
1912         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1913                 pt_id = fwd_ports_ids[i];
1914                 port = &ports[pt_id];
1915                 map_port_queue_stats_mapping_registers(pt_id, port);
1916         }
1917         if (with_tx_first) {
1918                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1919                 if (port_fwd_begin != NULL) {
1920                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1921                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1922                 }
1923                 while (with_tx_first--) {
1924                         launch_packet_forwarding(
1925                                         run_one_txonly_burst_on_core);
1926                         rte_eal_mp_wait_lcore();
1927                 }
1928                 port_fwd_end = tx_only_engine.port_fwd_end;
1929                 if (port_fwd_end != NULL) {
1930                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1931                                 (*port_fwd_end)(fwd_ports_ids[i]);
1932                 }
1933         }
1934         launch_packet_forwarding(start_pkt_forward_on_core);
1935 }
1936
1937 void
1938 stop_packet_forwarding(void)
1939 {
1940         port_fwd_end_t port_fwd_end;
1941         lcoreid_t lc_id;
1942         portid_t pt_id;
1943         int i;
1944
1945         if (test_done) {
1946                 printf("Packet forwarding not started\n");
1947                 return;
1948         }
1949         printf("Telling cores to stop...");
1950         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1951                 fwd_lcores[lc_id]->stopped = 1;
1952         printf("\nWaiting for lcores to finish...\n");
1953         rte_eal_mp_wait_lcore();
1954         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1955         if (port_fwd_end != NULL) {
1956                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1957                         pt_id = fwd_ports_ids[i];
1958                         (*port_fwd_end)(pt_id);
1959                 }
1960         }
1961
1962         fwd_stats_display();
1963
1964         printf("\nDone.\n");
1965         test_done = 1;
1966 }
1967
1968 void
1969 dev_set_link_up(portid_t pid)
1970 {
1971         if (rte_eth_dev_set_link_up(pid) < 0)
1972                 printf("\nSet link up fail.\n");
1973 }
1974
1975 void
1976 dev_set_link_down(portid_t pid)
1977 {
1978         if (rte_eth_dev_set_link_down(pid) < 0)
1979                 printf("\nSet link down fail.\n");
1980 }
1981
1982 static int
1983 all_ports_started(void)
1984 {
1985         portid_t pi;
1986         struct rte_port *port;
1987
1988         RTE_ETH_FOREACH_DEV(pi) {
1989                 port = &ports[pi];
1990                 /* Check if there is a port which is not started */
1991                 if ((port->port_status != RTE_PORT_STARTED) &&
1992                         (port->slave_flag == 0))
1993                         return 0;
1994         }
1995
1996         /* No port is not started */
1997         return 1;
1998 }
1999
2000 int
2001 port_is_stopped(portid_t port_id)
2002 {
2003         struct rte_port *port = &ports[port_id];
2004
2005         if ((port->port_status != RTE_PORT_STOPPED) &&
2006             (port->slave_flag == 0))
2007                 return 0;
2008         return 1;
2009 }
2010
2011 int
2012 all_ports_stopped(void)
2013 {
2014         portid_t pi;
2015
2016         RTE_ETH_FOREACH_DEV(pi) {
2017                 if (!port_is_stopped(pi))
2018                         return 0;
2019         }
2020
2021         return 1;
2022 }
2023
2024 int
2025 port_is_started(portid_t port_id)
2026 {
2027         if (port_id_is_invalid(port_id, ENABLED_WARN))
2028                 return 0;
2029
2030         if (ports[port_id].port_status != RTE_PORT_STARTED)
2031                 return 0;
2032
2033         return 1;
2034 }
2035
2036 int
2037 start_port(portid_t pid)
2038 {
2039         int diag, need_check_link_status = -1;
2040         portid_t pi;
2041         queueid_t qi;
2042         struct rte_port *port;
2043         struct rte_ether_addr mac_addr;
2044
2045         if (port_id_is_invalid(pid, ENABLED_WARN))
2046                 return 0;
2047
2048         if(dcb_config)
2049                 dcb_test = 1;
2050         RTE_ETH_FOREACH_DEV(pi) {
2051                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2052                         continue;
2053
2054                 need_check_link_status = 0;
2055                 port = &ports[pi];
2056                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2057                                                  RTE_PORT_HANDLING) == 0) {
2058                         printf("Port %d is now not stopped\n", pi);
2059                         continue;
2060                 }
2061
2062                 if (port->need_reconfig > 0) {
2063                         port->need_reconfig = 0;
2064
2065                         if (flow_isolate_all) {
2066                                 int ret = port_flow_isolate(pi, 1);
2067                                 if (ret) {
2068                                         printf("Failed to apply isolated"
2069                                                " mode on port %d\n", pi);
2070                                         return -1;
2071                                 }
2072                         }
2073                         configure_rxtx_dump_callbacks(0);
2074                         printf("Configuring Port %d (socket %u)\n", pi,
2075                                         port->socket_id);
2076                         /* configure port */
2077                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
2078                                                 &(port->dev_conf));
2079                         if (diag != 0) {
2080                                 if (rte_atomic16_cmpset(&(port->port_status),
2081                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2082                                         printf("Port %d can not be set back "
2083                                                         "to stopped\n", pi);
2084                                 printf("Fail to configure port %d\n", pi);
2085                                 /* try to reconfigure port next time */
2086                                 port->need_reconfig = 1;
2087                                 return -1;
2088                         }
2089                 }
2090                 if (port->need_reconfig_queues > 0) {
2091                         port->need_reconfig_queues = 0;
2092                         /* setup tx queues */
2093                         for (qi = 0; qi < nb_txq; qi++) {
2094                                 if ((numa_support) &&
2095                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2096                                         diag = rte_eth_tx_queue_setup(pi, qi,
2097                                                 port->nb_tx_desc[qi],
2098                                                 txring_numa[pi],
2099                                                 &(port->tx_conf[qi]));
2100                                 else
2101                                         diag = rte_eth_tx_queue_setup(pi, qi,
2102                                                 port->nb_tx_desc[qi],
2103                                                 port->socket_id,
2104                                                 &(port->tx_conf[qi]));
2105
2106                                 if (diag == 0)
2107                                         continue;
2108
2109                                 /* Fail to setup tx queue, return */
2110                                 if (rte_atomic16_cmpset(&(port->port_status),
2111                                                         RTE_PORT_HANDLING,
2112                                                         RTE_PORT_STOPPED) == 0)
2113                                         printf("Port %d can not be set back "
2114                                                         "to stopped\n", pi);
2115                                 printf("Fail to configure port %d tx queues\n",
2116                                        pi);
2117                                 /* try to reconfigure queues next time */
2118                                 port->need_reconfig_queues = 1;
2119                                 return -1;
2120                         }
2121                         for (qi = 0; qi < nb_rxq; qi++) {
2122                                 /* setup rx queues */
2123                                 if ((numa_support) &&
2124                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2125                                         struct rte_mempool * mp =
2126                                                 mbuf_pool_find(rxring_numa[pi]);
2127                                         if (mp == NULL) {
2128                                                 printf("Failed to setup RX queue:"
2129                                                         "No mempool allocation"
2130                                                         " on the socket %d\n",
2131                                                         rxring_numa[pi]);
2132                                                 return -1;
2133                                         }
2134
2135                                         diag = rte_eth_rx_queue_setup(pi, qi,
2136                                              port->nb_rx_desc[qi],
2137                                              rxring_numa[pi],
2138                                              &(port->rx_conf[qi]),
2139                                              mp);
2140                                 } else {
2141                                         struct rte_mempool *mp =
2142                                                 mbuf_pool_find(port->socket_id);
2143                                         if (mp == NULL) {
2144                                                 printf("Failed to setup RX queue:"
2145                                                         "No mempool allocation"
2146                                                         " on the socket %d\n",
2147                                                         port->socket_id);
2148                                                 return -1;
2149                                         }
2150                                         diag = rte_eth_rx_queue_setup(pi, qi,
2151                                              port->nb_rx_desc[qi],
2152                                              port->socket_id,
2153                                              &(port->rx_conf[qi]),
2154                                              mp);
2155                                 }
2156                                 if (diag == 0)
2157                                         continue;
2158
2159                                 /* Fail to setup rx queue, return */
2160                                 if (rte_atomic16_cmpset(&(port->port_status),
2161                                                         RTE_PORT_HANDLING,
2162                                                         RTE_PORT_STOPPED) == 0)
2163                                         printf("Port %d can not be set back "
2164                                                         "to stopped\n", pi);
2165                                 printf("Fail to configure port %d rx queues\n",
2166                                        pi);
2167                                 /* try to reconfigure queues next time */
2168                                 port->need_reconfig_queues = 1;
2169                                 return -1;
2170                         }
2171                 }
2172                 configure_rxtx_dump_callbacks(verbose_level);
2173                 /* start port */
2174                 if (rte_eth_dev_start(pi) < 0) {
2175                         printf("Fail to start port %d\n", pi);
2176
2177                         /* Fail to setup rx queue, return */
2178                         if (rte_atomic16_cmpset(&(port->port_status),
2179                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2180                                 printf("Port %d can not be set back to "
2181                                                         "stopped\n", pi);
2182                         continue;
2183                 }
2184
2185                 if (rte_atomic16_cmpset(&(port->port_status),
2186                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2187                         printf("Port %d can not be set into started\n", pi);
2188
2189                 rte_eth_macaddr_get(pi, &mac_addr);
2190                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2191                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2192                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2193                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2194
2195                 /* at least one port started, need checking link status */
2196                 need_check_link_status = 1;
2197         }
2198
2199         if (need_check_link_status == 1 && !no_link_check)
2200                 check_all_ports_link_status(RTE_PORT_ALL);
2201         else if (need_check_link_status == 0)
2202                 printf("Please stop the ports first\n");
2203
2204         printf("Done\n");
2205         return 0;
2206 }
2207
2208 void
2209 stop_port(portid_t pid)
2210 {
2211         portid_t pi;
2212         struct rte_port *port;
2213         int need_check_link_status = 0;
2214
2215         if (dcb_test) {
2216                 dcb_test = 0;
2217                 dcb_config = 0;
2218         }
2219
2220         if (port_id_is_invalid(pid, ENABLED_WARN))
2221                 return;
2222
2223         printf("Stopping ports...\n");
2224
2225         RTE_ETH_FOREACH_DEV(pi) {
2226                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2227                         continue;
2228
2229                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2230                         printf("Please remove port %d from forwarding configuration.\n", pi);
2231                         continue;
2232                 }
2233
2234                 if (port_is_bonding_slave(pi)) {
2235                         printf("Please remove port %d from bonded device.\n", pi);
2236                         continue;
2237                 }
2238
2239                 port = &ports[pi];
2240                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2241                                                 RTE_PORT_HANDLING) == 0)
2242                         continue;
2243
2244                 rte_eth_dev_stop(pi);
2245
2246                 if (rte_atomic16_cmpset(&(port->port_status),
2247                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2248                         printf("Port %d can not be set into stopped\n", pi);
2249                 need_check_link_status = 1;
2250         }
2251         if (need_check_link_status && !no_link_check)
2252                 check_all_ports_link_status(RTE_PORT_ALL);
2253
2254         printf("Done\n");
2255 }
2256
2257 static void
2258 remove_invalid_ports_in(portid_t *array, portid_t *total)
2259 {
2260         portid_t i;
2261         portid_t new_total = 0;
2262
2263         for (i = 0; i < *total; i++)
2264                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2265                         array[new_total] = array[i];
2266                         new_total++;
2267                 }
2268         *total = new_total;
2269 }
2270
2271 static void
2272 remove_invalid_ports(void)
2273 {
2274         remove_invalid_ports_in(ports_ids, &nb_ports);
2275         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2276         nb_cfg_ports = nb_fwd_ports;
2277 }
2278
2279 void
2280 close_port(portid_t pid)
2281 {
2282         portid_t pi;
2283         struct rte_port *port;
2284
2285         if (port_id_is_invalid(pid, ENABLED_WARN))
2286                 return;
2287
2288         printf("Closing ports...\n");
2289
2290         RTE_ETH_FOREACH_DEV(pi) {
2291                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2292                         continue;
2293
2294                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2295                         printf("Please remove port %d from forwarding configuration.\n", pi);
2296                         continue;
2297                 }
2298
2299                 if (port_is_bonding_slave(pi)) {
2300                         printf("Please remove port %d from bonded device.\n", pi);
2301                         continue;
2302                 }
2303
2304                 port = &ports[pi];
2305                 if (rte_atomic16_cmpset(&(port->port_status),
2306                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2307                         printf("Port %d is already closed\n", pi);
2308                         continue;
2309                 }
2310
2311                 if (rte_atomic16_cmpset(&(port->port_status),
2312                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2313                         printf("Port %d is now not stopped\n", pi);
2314                         continue;
2315                 }
2316
2317                 if (port->flow_list)
2318                         port_flow_flush(pi);
2319                 rte_eth_dev_close(pi);
2320
2321                 remove_invalid_ports();
2322
2323                 if (rte_atomic16_cmpset(&(port->port_status),
2324                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2325                         printf("Port %d cannot be set to closed\n", pi);
2326         }
2327
2328         printf("Done\n");
2329 }
2330
2331 void
2332 reset_port(portid_t pid)
2333 {
2334         int diag;
2335         portid_t pi;
2336         struct rte_port *port;
2337
2338         if (port_id_is_invalid(pid, ENABLED_WARN))
2339                 return;
2340
2341         printf("Resetting ports...\n");
2342
2343         RTE_ETH_FOREACH_DEV(pi) {
2344                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2345                         continue;
2346
2347                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2348                         printf("Please remove port %d from forwarding "
2349                                "configuration.\n", pi);
2350                         continue;
2351                 }
2352
2353                 if (port_is_bonding_slave(pi)) {
2354                         printf("Please remove port %d from bonded device.\n",
2355                                pi);
2356                         continue;
2357                 }
2358
2359                 diag = rte_eth_dev_reset(pi);
2360                 if (diag == 0) {
2361                         port = &ports[pi];
2362                         port->need_reconfig = 1;
2363                         port->need_reconfig_queues = 1;
2364                 } else {
2365                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2366                 }
2367         }
2368
2369         printf("Done\n");
2370 }
2371
2372 void
2373 attach_port(char *identifier)
2374 {
2375         portid_t pi;
2376         struct rte_dev_iterator iterator;
2377
2378         printf("Attaching a new port...\n");
2379
2380         if (identifier == NULL) {
2381                 printf("Invalid parameters are specified\n");
2382                 return;
2383         }
2384
2385         if (rte_dev_probe(identifier) != 0) {
2386                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2387                 return;
2388         }
2389
2390         /* first attach mode: event */
2391         if (setup_on_probe_event) {
2392                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2393                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2394                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2395                                         ports[pi].need_setup != 0)
2396                                 setup_attached_port(pi);
2397                 return;
2398         }
2399
2400         /* second attach mode: iterator */
2401         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2402                 /* setup ports matching the devargs used for probing */
2403                 if (port_is_forwarding(pi))
2404                         continue; /* port was already attached before */
2405                 setup_attached_port(pi);
2406         }
2407 }
2408
2409 static void
2410 setup_attached_port(portid_t pi)
2411 {
2412         unsigned int socket_id;
2413
2414         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2415         /* if socket_id is invalid, set to the first available socket. */
2416         if (check_socket_id(socket_id) < 0)
2417                 socket_id = socket_ids[0];
2418         reconfig(pi, socket_id);
2419         rte_eth_promiscuous_enable(pi);
2420
2421         ports_ids[nb_ports++] = pi;
2422         fwd_ports_ids[nb_fwd_ports++] = pi;
2423         nb_cfg_ports = nb_fwd_ports;
2424         ports[pi].need_setup = 0;
2425         ports[pi].port_status = RTE_PORT_STOPPED;
2426
2427         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2428         printf("Done\n");
2429 }
2430
2431 void
2432 detach_port_device(portid_t port_id)
2433 {
2434         struct rte_device *dev;
2435         portid_t sibling;
2436
2437         printf("Removing a device...\n");
2438
2439         dev = rte_eth_devices[port_id].device;
2440         if (dev == NULL) {
2441                 printf("Device already removed\n");
2442                 return;
2443         }
2444
2445         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2446                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2447                         printf("Port not stopped\n");
2448                         return;
2449                 }
2450                 printf("Port was not closed\n");
2451                 if (ports[port_id].flow_list)
2452                         port_flow_flush(port_id);
2453         }
2454
2455         if (rte_dev_remove(dev) != 0) {
2456                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2457                 return;
2458         }
2459         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2460                 /* reset mapping between old ports and removed device */
2461                 rte_eth_devices[sibling].device = NULL;
2462                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2463                         /* sibling ports are forced to be closed */
2464                         ports[sibling].port_status = RTE_PORT_CLOSED;
2465                         printf("Port %u is closed\n", sibling);
2466                 }
2467         }
2468
2469         remove_invalid_ports();
2470
2471         printf("Device of port %u is detached\n", port_id);
2472         printf("Now total ports is %d\n", nb_ports);
2473         printf("Done\n");
2474         return;
2475 }
2476
2477 void
2478 pmd_test_exit(void)
2479 {
2480         struct rte_device *device;
2481         portid_t pt_id;
2482         int ret;
2483         int i;
2484
2485         if (test_done == 0)
2486                 stop_packet_forwarding();
2487
2488         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2489                 if (mempools[i]) {
2490                         if (mp_alloc_type == MP_ALLOC_ANON)
2491                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2492                                                      NULL);
2493                 }
2494         }
2495         if (ports != NULL) {
2496                 no_link_check = 1;
2497                 RTE_ETH_FOREACH_DEV(pt_id) {
2498                         printf("\nStopping port %d...\n", pt_id);
2499                         fflush(stdout);
2500                         stop_port(pt_id);
2501                 }
2502                 RTE_ETH_FOREACH_DEV(pt_id) {
2503                         printf("\nShutting down port %d...\n", pt_id);
2504                         fflush(stdout);
2505                         close_port(pt_id);
2506
2507                         /*
2508                          * This is a workaround to fix a virtio-user issue that
2509                          * requires to call clean-up routine to remove existing
2510                          * socket.
2511                          * This workaround valid only for testpmd, needs a fix
2512                          * valid for all applications.
2513                          * TODO: Implement proper resource cleanup
2514                          */
2515                         device = rte_eth_devices[pt_id].device;
2516                         if (device && !strcmp(device->driver->name, "net_virtio_user"))
2517                                 detach_port_device(pt_id);
2518                 }
2519         }
2520
2521         if (hot_plug) {
2522                 ret = rte_dev_event_monitor_stop();
2523                 if (ret) {
2524                         RTE_LOG(ERR, EAL,
2525                                 "fail to stop device event monitor.");
2526                         return;
2527                 }
2528
2529                 ret = rte_dev_event_callback_unregister(NULL,
2530                         dev_event_callback, NULL);
2531                 if (ret < 0) {
2532                         RTE_LOG(ERR, EAL,
2533                                 "fail to unregister device event callback.\n");
2534                         return;
2535                 }
2536
2537                 ret = rte_dev_hotplug_handle_disable();
2538                 if (ret) {
2539                         RTE_LOG(ERR, EAL,
2540                                 "fail to disable hotplug handling.\n");
2541                         return;
2542                 }
2543         }
2544         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2545                 if (mempools[i])
2546                         rte_mempool_free(mempools[i]);
2547         }
2548
2549         printf("\nBye...\n");
2550 }
2551
2552 typedef void (*cmd_func_t)(void);
2553 struct pmd_test_command {
2554         const char *cmd_name;
2555         cmd_func_t cmd_func;
2556 };
2557
2558 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2559
2560 /* Check the link status of all ports in up to 9s, and print them finally */
2561 static void
2562 check_all_ports_link_status(uint32_t port_mask)
2563 {
2564 #define CHECK_INTERVAL 100 /* 100ms */
2565 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2566         portid_t portid;
2567         uint8_t count, all_ports_up, print_flag = 0;
2568         struct rte_eth_link link;
2569
2570         printf("Checking link statuses...\n");
2571         fflush(stdout);
2572         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2573                 all_ports_up = 1;
2574                 RTE_ETH_FOREACH_DEV(portid) {
2575                         if ((port_mask & (1 << portid)) == 0)
2576                                 continue;
2577                         memset(&link, 0, sizeof(link));
2578                         rte_eth_link_get_nowait(portid, &link);
2579                         /* print link status if flag set */
2580                         if (print_flag == 1) {
2581                                 if (link.link_status)
2582                                         printf(
2583                                         "Port%d Link Up. speed %u Mbps- %s\n",
2584                                         portid, link.link_speed,
2585                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2586                                         ("full-duplex") : ("half-duplex\n"));
2587                                 else
2588                                         printf("Port %d Link Down\n", portid);
2589                                 continue;
2590                         }
2591                         /* clear all_ports_up flag if any link down */
2592                         if (link.link_status == ETH_LINK_DOWN) {
2593                                 all_ports_up = 0;
2594                                 break;
2595                         }
2596                 }
2597                 /* after finally printing all link status, get out */
2598                 if (print_flag == 1)
2599                         break;
2600
2601                 if (all_ports_up == 0) {
2602                         fflush(stdout);
2603                         rte_delay_ms(CHECK_INTERVAL);
2604                 }
2605
2606                 /* set the print_flag if all ports up or timeout */
2607                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2608                         print_flag = 1;
2609                 }
2610
2611                 if (lsc_interrupt)
2612                         break;
2613         }
2614 }
2615
2616 /*
2617  * This callback is for remove a port for a device. It has limitation because
2618  * it is not for multiple port removal for a device.
2619  * TODO: the device detach invoke will plan to be removed from user side to
2620  * eal. And convert all PMDs to free port resources on ether device closing.
2621  */
2622 static void
2623 rmv_port_callback(void *arg)
2624 {
2625         int need_to_start = 0;
2626         int org_no_link_check = no_link_check;
2627         portid_t port_id = (intptr_t)arg;
2628
2629         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2630
2631         if (!test_done && port_is_forwarding(port_id)) {
2632                 need_to_start = 1;
2633                 stop_packet_forwarding();
2634         }
2635         no_link_check = 1;
2636         stop_port(port_id);
2637         no_link_check = org_no_link_check;
2638         close_port(port_id);
2639         detach_port_device(port_id);
2640         if (need_to_start)
2641                 start_packet_forwarding(0);
2642 }
2643
2644 /* This function is used by the interrupt thread */
2645 static int
2646 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2647                   void *ret_param)
2648 {
2649         RTE_SET_USED(param);
2650         RTE_SET_USED(ret_param);
2651
2652         if (type >= RTE_ETH_EVENT_MAX) {
2653                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2654                         port_id, __func__, type);
2655                 fflush(stderr);
2656         } else if (event_print_mask & (UINT32_C(1) << type)) {
2657                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2658                         eth_event_desc[type]);
2659                 fflush(stdout);
2660         }
2661
2662         switch (type) {
2663         case RTE_ETH_EVENT_NEW:
2664                 ports[port_id].need_setup = 1;
2665                 ports[port_id].port_status = RTE_PORT_HANDLING;
2666                 break;
2667         case RTE_ETH_EVENT_INTR_RMV:
2668                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2669                         break;
2670                 if (rte_eal_alarm_set(100000,
2671                                 rmv_port_callback, (void *)(intptr_t)port_id))
2672                         fprintf(stderr, "Could not set up deferred device removal\n");
2673                 break;
2674         default:
2675                 break;
2676         }
2677         return 0;
2678 }
2679
2680 static int
2681 register_eth_event_callback(void)
2682 {
2683         int ret;
2684         enum rte_eth_event_type event;
2685
2686         for (event = RTE_ETH_EVENT_UNKNOWN;
2687                         event < RTE_ETH_EVENT_MAX; event++) {
2688                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2689                                 event,
2690                                 eth_event_callback,
2691                                 NULL);
2692                 if (ret != 0) {
2693                         TESTPMD_LOG(ERR, "Failed to register callback for "
2694                                         "%s event\n", eth_event_desc[event]);
2695                         return -1;
2696                 }
2697         }
2698
2699         return 0;
2700 }
2701
2702 /* This function is used by the interrupt thread */
2703 static void
2704 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2705                              __rte_unused void *arg)
2706 {
2707         uint16_t port_id;
2708         int ret;
2709
2710         if (type >= RTE_DEV_EVENT_MAX) {
2711                 fprintf(stderr, "%s called upon invalid event %d\n",
2712                         __func__, type);
2713                 fflush(stderr);
2714         }
2715
2716         switch (type) {
2717         case RTE_DEV_EVENT_REMOVE:
2718                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2719                         device_name);
2720                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2721                 if (ret) {
2722                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2723                                 device_name);
2724                         return;
2725                 }
2726                 /*
2727                  * Because the user's callback is invoked in eal interrupt
2728                  * callback, the interrupt callback need to be finished before
2729                  * it can be unregistered when detaching device. So finish
2730                  * callback soon and use a deferred removal to detach device
2731                  * is need. It is a workaround, once the device detaching be
2732                  * moved into the eal in the future, the deferred removal could
2733                  * be deleted.
2734                  */
2735                 if (rte_eal_alarm_set(100000,
2736                                 rmv_port_callback, (void *)(intptr_t)port_id))
2737                         RTE_LOG(ERR, EAL,
2738                                 "Could not set up deferred device removal\n");
2739                 break;
2740         case RTE_DEV_EVENT_ADD:
2741                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2742                         device_name);
2743                 /* TODO: After finish kernel driver binding,
2744                  * begin to attach port.
2745                  */
2746                 break;
2747         default:
2748                 break;
2749         }
2750 }
2751
2752 static int
2753 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2754 {
2755         uint16_t i;
2756         int diag;
2757         uint8_t mapping_found = 0;
2758
2759         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2760                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2761                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2762                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2763                                         tx_queue_stats_mappings[i].queue_id,
2764                                         tx_queue_stats_mappings[i].stats_counter_id);
2765                         if (diag != 0)
2766                                 return diag;
2767                         mapping_found = 1;
2768                 }
2769         }
2770         if (mapping_found)
2771                 port->tx_queue_stats_mapping_enabled = 1;
2772         return 0;
2773 }
2774
2775 static int
2776 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2777 {
2778         uint16_t i;
2779         int diag;
2780         uint8_t mapping_found = 0;
2781
2782         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2783                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2784                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2785                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2786                                         rx_queue_stats_mappings[i].queue_id,
2787                                         rx_queue_stats_mappings[i].stats_counter_id);
2788                         if (diag != 0)
2789                                 return diag;
2790                         mapping_found = 1;
2791                 }
2792         }
2793         if (mapping_found)
2794                 port->rx_queue_stats_mapping_enabled = 1;
2795         return 0;
2796 }
2797
2798 static void
2799 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2800 {
2801         int diag = 0;
2802
2803         diag = set_tx_queue_stats_mapping_registers(pi, port);
2804         if (diag != 0) {
2805                 if (diag == -ENOTSUP) {
2806                         port->tx_queue_stats_mapping_enabled = 0;
2807                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2808                 }
2809                 else
2810                         rte_exit(EXIT_FAILURE,
2811                                         "set_tx_queue_stats_mapping_registers "
2812                                         "failed for port id=%d diag=%d\n",
2813                                         pi, diag);
2814         }
2815
2816         diag = set_rx_queue_stats_mapping_registers(pi, port);
2817         if (diag != 0) {
2818                 if (diag == -ENOTSUP) {
2819                         port->rx_queue_stats_mapping_enabled = 0;
2820                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2821                 }
2822                 else
2823                         rte_exit(EXIT_FAILURE,
2824                                         "set_rx_queue_stats_mapping_registers "
2825                                         "failed for port id=%d diag=%d\n",
2826                                         pi, diag);
2827         }
2828 }
2829
2830 static void
2831 rxtx_port_config(struct rte_port *port)
2832 {
2833         uint16_t qid;
2834         uint64_t offloads;
2835
2836         for (qid = 0; qid < nb_rxq; qid++) {
2837                 offloads = port->rx_conf[qid].offloads;
2838                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2839                 port->rx_conf[qid].offloads |= offloads;
2840
2841                 /* Check if any Rx parameters have been passed */
2842                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2843                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2844
2845                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2846                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2847
2848                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2849                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2850
2851                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2852                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2853
2854                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2855                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2856
2857                 port->nb_rx_desc[qid] = nb_rxd;
2858         }
2859
2860         for (qid = 0; qid < nb_txq; qid++) {
2861                 offloads = port->tx_conf[qid].offloads;
2862                 port->tx_conf[qid] = port->dev_info.default_txconf;
2863                 port->tx_conf[qid].offloads |= offloads;
2864
2865                 /* Check if any Tx parameters have been passed */
2866                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2867                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2868
2869                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2870                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2871
2872                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2873                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2874
2875                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2876                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2877
2878                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2879                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2880
2881                 port->nb_tx_desc[qid] = nb_txd;
2882         }
2883 }
2884
2885 void
2886 init_port_config(void)
2887 {
2888         portid_t pid;
2889         struct rte_port *port;
2890
2891         RTE_ETH_FOREACH_DEV(pid) {
2892                 port = &ports[pid];
2893                 port->dev_conf.fdir_conf = fdir_conf;
2894                 rte_eth_dev_info_get(pid, &port->dev_info);
2895                 if (nb_rxq > 1) {
2896                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2897                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2898                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2899                 } else {
2900                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2901                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2902                 }
2903
2904                 if (port->dcb_flag == 0) {
2905                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2906                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2907                         else
2908                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2909                 }
2910
2911                 rxtx_port_config(port);
2912
2913                 rte_eth_macaddr_get(pid, &port->eth_addr);
2914
2915                 map_port_queue_stats_mapping_registers(pid, port);
2916 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2917                 rte_pmd_ixgbe_bypass_init(pid);
2918 #endif
2919
2920                 if (lsc_interrupt &&
2921                     (rte_eth_devices[pid].data->dev_flags &
2922                      RTE_ETH_DEV_INTR_LSC))
2923                         port->dev_conf.intr_conf.lsc = 1;
2924                 if (rmv_interrupt &&
2925                     (rte_eth_devices[pid].data->dev_flags &
2926                      RTE_ETH_DEV_INTR_RMV))
2927                         port->dev_conf.intr_conf.rmv = 1;
2928         }
2929 }
2930
2931 void set_port_slave_flag(portid_t slave_pid)
2932 {
2933         struct rte_port *port;
2934
2935         port = &ports[slave_pid];
2936         port->slave_flag = 1;
2937 }
2938
2939 void clear_port_slave_flag(portid_t slave_pid)
2940 {
2941         struct rte_port *port;
2942
2943         port = &ports[slave_pid];
2944         port->slave_flag = 0;
2945 }
2946
2947 uint8_t port_is_bonding_slave(portid_t slave_pid)
2948 {
2949         struct rte_port *port;
2950
2951         port = &ports[slave_pid];
2952         if ((rte_eth_devices[slave_pid].data->dev_flags &
2953             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
2954                 return 1;
2955         return 0;
2956 }
2957
2958 const uint16_t vlan_tags[] = {
2959                 0,  1,  2,  3,  4,  5,  6,  7,
2960                 8,  9, 10, 11,  12, 13, 14, 15,
2961                 16, 17, 18, 19, 20, 21, 22, 23,
2962                 24, 25, 26, 27, 28, 29, 30, 31
2963 };
2964
2965 static  int
2966 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
2967                  enum dcb_mode_enable dcb_mode,
2968                  enum rte_eth_nb_tcs num_tcs,
2969                  uint8_t pfc_en)
2970 {
2971         uint8_t i;
2972         int32_t rc;
2973         struct rte_eth_rss_conf rss_conf;
2974
2975         /*
2976          * Builds up the correct configuration for dcb+vt based on the vlan tags array
2977          * given above, and the number of traffic classes available for use.
2978          */
2979         if (dcb_mode == DCB_VT_ENABLED) {
2980                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
2981                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
2982                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2983                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
2984
2985                 /* VMDQ+DCB RX and TX configurations */
2986                 vmdq_rx_conf->enable_default_pool = 0;
2987                 vmdq_rx_conf->default_pool = 0;
2988                 vmdq_rx_conf->nb_queue_pools =
2989                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2990                 vmdq_tx_conf->nb_queue_pools =
2991                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2992
2993                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
2994                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
2995                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
2996                         vmdq_rx_conf->pool_map[i].pools =
2997                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
2998                 }
2999                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3000                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3001                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3002                 }
3003
3004                 /* set DCB mode of RX and TX of multiple queues */
3005                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
3006                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3007         } else {
3008                 struct rte_eth_dcb_rx_conf *rx_conf =
3009                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3010                 struct rte_eth_dcb_tx_conf *tx_conf =
3011                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3012
3013                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3014                 if (rc != 0)
3015                         return rc;
3016
3017                 rx_conf->nb_tcs = num_tcs;
3018                 tx_conf->nb_tcs = num_tcs;
3019
3020                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3021                         rx_conf->dcb_tc[i] = i % num_tcs;
3022                         tx_conf->dcb_tc[i] = i % num_tcs;
3023                 }
3024
3025                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
3026                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3027                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3028         }
3029
3030         if (pfc_en)
3031                 eth_conf->dcb_capability_en =
3032                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3033         else
3034                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3035
3036         return 0;
3037 }
3038
3039 int
3040 init_port_dcb_config(portid_t pid,
3041                      enum dcb_mode_enable dcb_mode,
3042                      enum rte_eth_nb_tcs num_tcs,
3043                      uint8_t pfc_en)
3044 {
3045         struct rte_eth_conf port_conf;
3046         struct rte_port *rte_port;
3047         int retval;
3048         uint16_t i;
3049
3050         rte_port = &ports[pid];
3051
3052         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3053         /* Enter DCB configuration status */
3054         dcb_config = 1;
3055
3056         port_conf.rxmode = rte_port->dev_conf.rxmode;
3057         port_conf.txmode = rte_port->dev_conf.txmode;
3058
3059         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3060         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3061         if (retval < 0)
3062                 return retval;
3063         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3064
3065         /* re-configure the device . */
3066         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3067         if (retval < 0)
3068                 return retval;
3069         rte_eth_dev_info_get(pid, &rte_port->dev_info);
3070
3071         /* If dev_info.vmdq_pool_base is greater than 0,
3072          * the queue id of vmdq pools is started after pf queues.
3073          */
3074         if (dcb_mode == DCB_VT_ENABLED &&
3075             rte_port->dev_info.vmdq_pool_base > 0) {
3076                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3077                         " for port %d.", pid);
3078                 return -1;
3079         }
3080
3081         /* Assume the ports in testpmd have the same dcb capability
3082          * and has the same number of rxq and txq in dcb mode
3083          */
3084         if (dcb_mode == DCB_VT_ENABLED) {
3085                 if (rte_port->dev_info.max_vfs > 0) {
3086                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3087                         nb_txq = rte_port->dev_info.nb_tx_queues;
3088                 } else {
3089                         nb_rxq = rte_port->dev_info.max_rx_queues;
3090                         nb_txq = rte_port->dev_info.max_tx_queues;
3091                 }
3092         } else {
3093                 /*if vt is disabled, use all pf queues */
3094                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3095                         nb_rxq = rte_port->dev_info.max_rx_queues;
3096                         nb_txq = rte_port->dev_info.max_tx_queues;
3097                 } else {
3098                         nb_rxq = (queueid_t)num_tcs;
3099                         nb_txq = (queueid_t)num_tcs;
3100
3101                 }
3102         }
3103         rx_free_thresh = 64;
3104
3105         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3106
3107         rxtx_port_config(rte_port);
3108         /* VLAN filter */
3109         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3110         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3111                 rx_vft_set(pid, vlan_tags[i], 1);
3112
3113         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
3114         map_port_queue_stats_mapping_registers(pid, rte_port);
3115
3116         rte_port->dcb_flag = 1;
3117
3118         return 0;
3119 }
3120
3121 static void
3122 init_port(void)
3123 {
3124         /* Configuration of Ethernet ports. */
3125         ports = rte_zmalloc("testpmd: ports",
3126                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3127                             RTE_CACHE_LINE_SIZE);
3128         if (ports == NULL) {
3129                 rte_exit(EXIT_FAILURE,
3130                                 "rte_zmalloc(%d struct rte_port) failed\n",
3131                                 RTE_MAX_ETHPORTS);
3132         }
3133
3134         /* Initialize ports NUMA structures */
3135         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3136         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3137         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3138 }
3139
3140 static void
3141 force_quit(void)
3142 {
3143         pmd_test_exit();
3144         prompt_exit();
3145 }
3146
3147 static void
3148 print_stats(void)
3149 {
3150         uint8_t i;
3151         const char clr[] = { 27, '[', '2', 'J', '\0' };
3152         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3153
3154         /* Clear screen and move to top left */
3155         printf("%s%s", clr, top_left);
3156
3157         printf("\nPort statistics ====================================");
3158         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3159                 nic_stats_display(fwd_ports_ids[i]);
3160
3161         fflush(stdout);
3162 }
3163
3164 static void
3165 signal_handler(int signum)
3166 {
3167         if (signum == SIGINT || signum == SIGTERM) {
3168                 printf("\nSignal %d received, preparing to exit...\n",
3169                                 signum);
3170 #ifdef RTE_LIBRTE_PDUMP
3171                 /* uninitialize packet capture framework */
3172                 rte_pdump_uninit();
3173 #endif
3174 #ifdef RTE_LIBRTE_LATENCY_STATS
3175                 rte_latencystats_uninit();
3176 #endif
3177                 force_quit();
3178                 /* Set flag to indicate the force termination. */
3179                 f_quit = 1;
3180                 /* exit with the expected status */
3181                 signal(signum, SIG_DFL);
3182                 kill(getpid(), signum);
3183         }
3184 }
3185
3186 int
3187 main(int argc, char** argv)
3188 {
3189         int diag;
3190         portid_t port_id;
3191         uint16_t count;
3192         int ret;
3193
3194         signal(SIGINT, signal_handler);
3195         signal(SIGTERM, signal_handler);
3196
3197         diag = rte_eal_init(argc, argv);
3198         if (diag < 0)
3199                 rte_panic("Cannot init EAL\n");
3200
3201         testpmd_logtype = rte_log_register("testpmd");
3202         if (testpmd_logtype < 0)
3203                 rte_panic("Cannot register log type");
3204         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3205
3206         ret = register_eth_event_callback();
3207         if (ret != 0)
3208                 rte_panic("Cannot register for ethdev events");
3209
3210 #ifdef RTE_LIBRTE_PDUMP
3211         /* initialize packet capture framework */
3212         rte_pdump_init();
3213 #endif
3214
3215         count = 0;
3216         RTE_ETH_FOREACH_DEV(port_id) {
3217                 ports_ids[count] = port_id;
3218                 count++;
3219         }
3220         nb_ports = (portid_t) count;
3221         if (nb_ports == 0)
3222                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3223
3224         /* allocate port structures, and init them */
3225         init_port();
3226
3227         set_def_fwd_config();
3228         if (nb_lcores == 0)
3229                 rte_panic("Empty set of forwarding logical cores - check the "
3230                           "core mask supplied in the command parameters\n");
3231
3232         /* Bitrate/latency stats disabled by default */
3233 #ifdef RTE_LIBRTE_BITRATE
3234         bitrate_enabled = 0;
3235 #endif
3236 #ifdef RTE_LIBRTE_LATENCY_STATS
3237         latencystats_enabled = 0;
3238 #endif
3239
3240         /* on FreeBSD, mlockall() is disabled by default */
3241 #ifdef RTE_EXEC_ENV_FREEBSD
3242         do_mlockall = 0;
3243 #else
3244         do_mlockall = 1;
3245 #endif
3246
3247         argc -= diag;
3248         argv += diag;
3249         if (argc > 1)
3250                 launch_args_parse(argc, argv);
3251
3252         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3253                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3254                         strerror(errno));
3255         }
3256
3257         if (tx_first && interactive)
3258                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3259                                 "interactive mode.\n");
3260
3261         if (tx_first && lsc_interrupt) {
3262                 printf("Warning: lsc_interrupt needs to be off when "
3263                                 " using tx_first. Disabling.\n");
3264                 lsc_interrupt = 0;
3265         }
3266
3267         if (!nb_rxq && !nb_txq)
3268                 printf("Warning: Either rx or tx queues should be non-zero\n");
3269
3270         if (nb_rxq > 1 && nb_rxq > nb_txq)
3271                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3272                        "but nb_txq=%d will prevent to fully test it.\n",
3273                        nb_rxq, nb_txq);
3274
3275         init_config();
3276
3277         if (hot_plug) {
3278                 ret = rte_dev_hotplug_handle_enable();
3279                 if (ret) {
3280                         RTE_LOG(ERR, EAL,
3281                                 "fail to enable hotplug handling.");
3282                         return -1;
3283                 }
3284
3285                 ret = rte_dev_event_monitor_start();
3286                 if (ret) {
3287                         RTE_LOG(ERR, EAL,
3288                                 "fail to start device event monitoring.");
3289                         return -1;
3290                 }
3291
3292                 ret = rte_dev_event_callback_register(NULL,
3293                         dev_event_callback, NULL);
3294                 if (ret) {
3295                         RTE_LOG(ERR, EAL,
3296                                 "fail  to register device event callback\n");
3297                         return -1;
3298                 }
3299         }
3300
3301         if (start_port(RTE_PORT_ALL) != 0)
3302                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3303
3304         /* set all ports to promiscuous mode by default */
3305         RTE_ETH_FOREACH_DEV(port_id)
3306                 rte_eth_promiscuous_enable(port_id);
3307
3308         /* Init metrics library */
3309         rte_metrics_init(rte_socket_id());
3310
3311 #ifdef RTE_LIBRTE_LATENCY_STATS
3312         if (latencystats_enabled != 0) {
3313                 int ret = rte_latencystats_init(1, NULL);
3314                 if (ret)
3315                         printf("Warning: latencystats init()"
3316                                 " returned error %d\n", ret);
3317                 printf("Latencystats running on lcore %d\n",
3318                         latencystats_lcore_id);
3319         }
3320 #endif
3321
3322         /* Setup bitrate stats */
3323 #ifdef RTE_LIBRTE_BITRATE
3324         if (bitrate_enabled != 0) {
3325                 bitrate_data = rte_stats_bitrate_create();
3326                 if (bitrate_data == NULL)
3327                         rte_exit(EXIT_FAILURE,
3328                                 "Could not allocate bitrate data.\n");
3329                 rte_stats_bitrate_reg(bitrate_data);
3330         }
3331 #endif
3332
3333 #ifdef RTE_LIBRTE_CMDLINE
3334         if (strlen(cmdline_filename) != 0)
3335                 cmdline_read_from_file(cmdline_filename);
3336
3337         if (interactive == 1) {
3338                 if (auto_start) {
3339                         printf("Start automatic packet forwarding\n");
3340                         start_packet_forwarding(0);
3341                 }
3342                 prompt();
3343                 pmd_test_exit();
3344         } else
3345 #endif
3346         {
3347                 char c;
3348                 int rc;
3349
3350                 f_quit = 0;
3351
3352                 printf("No commandline core given, start packet forwarding\n");
3353                 start_packet_forwarding(tx_first);
3354                 if (stats_period != 0) {
3355                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3356                         uint64_t timer_period;
3357
3358                         /* Convert to number of cycles */
3359                         timer_period = stats_period * rte_get_timer_hz();
3360
3361                         while (f_quit == 0) {
3362                                 cur_time = rte_get_timer_cycles();
3363                                 diff_time += cur_time - prev_time;
3364
3365                                 if (diff_time >= timer_period) {
3366                                         print_stats();
3367                                         /* Reset the timer */
3368                                         diff_time = 0;
3369                                 }
3370                                 /* Sleep to avoid unnecessary checks */
3371                                 prev_time = cur_time;
3372                                 sleep(1);
3373                         }
3374                 }
3375
3376                 printf("Press enter to exit\n");
3377                 rc = read(0, &c, 1);
3378                 pmd_test_exit();
3379                 if (rc < 0)
3380                         return 1;
3381         }
3382
3383         return 0;
3384 }