app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_malloc_heap.h>
  31 #include <rte_memory.h>
  32 #include <rte_memcpy.h>
  33 #include <rte_launch.h>
  34 #include <rte_eal.h>
  35 #include <rte_alarm.h>
  36 #include <rte_per_lcore.h>
  37 #include <rte_lcore.h>
  38 #include <rte_atomic.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_LIBRTE_IXGBE_PMD
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIBRTE_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #include <rte_metrics.h>
  58 #ifdef RTE_LIBRTE_BITRATE
  59 #include <rte_bitrate.h>
  60 #endif
  61 #ifdef RTE_LIBRTE_LATENCY_STATS
  62 #include <rte_latencystats.h>
  63 #endif
  64
  65 #include "testpmd.h"
  66
  67 #ifndef MAP_HUGETLB
  68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  69 #define HUGE_FLAG (0x40000)
  70 #else
  71 #define HUGE_FLAG MAP_HUGETLB
  72 #endif
  73
  74 #ifndef MAP_HUGE_SHIFT
  75 /* older kernels (or FreeBSD) will not have this define */
  76 #define HUGE_SHIFT (26)
  77 #else
  78 #define HUGE_SHIFT MAP_HUGE_SHIFT
  79 #endif
  80
  81 #define EXTMEM_HEAP_NAME "extmem"
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192 uint16_t mempool_flags;
 193
 194 struct fwd_config cur_fwd_config;
 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 196 uint32_t retry_enabled;
 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 199
 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 201 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 202                                       * specified on command-line. */
 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 204
 205 /*
 206  * In container, it cannot terminate the process which running with 'stats-period'
 207  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 208  */
 209 uint8_t f_quit;
 210
 211 /*
 212  * Configuration of packet segments used by the "txonly" processing engine.
 213  */
 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 216         TXONLY_DEF_PACKET_LEN,
 217 };
 218 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 219
 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 221 /**< Split policy for packets to TX. */
 222
 223 uint8_t txonly_multi_flow;
 224 /**< Whether multiple flows are generated in TXONLY mode. */
 225
 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 228
 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 230 uint8_t dcb_config = 0;
 231
 232 /* Whether the dcb is in testing status */
 233 uint8_t dcb_test = 0;
 234
 235 /*
 236  * Configurable number of RX/TX queues.
 237  */
 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 240
 241 /*
 242  * Configurable number of RX/TX ring descriptors.
 243  * Defaults are supplied by drivers via ethdev.
 244  */
 245 #define RTE_TEST_RX_DESC_DEFAULT 0
 246 #define RTE_TEST_TX_DESC_DEFAULT 0
 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 249
 250 #define RTE_PMD_PARAM_UNSET -1
 251 /*
 252  * Configurable values of RX and TX ring threshold registers.
 253  */
 254
 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 258
 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 262
 263 /*
 264  * Configurable value of RX free threshold.
 265  */
 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 267
 268 /*
 269  * Configurable value of RX drop enable.
 270  */
 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 272
 273 /*
 274  * Configurable value of TX free threshold.
 275  */
 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of TX RS bit threshold.
 280  */
 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of buffered packets before sending.
 285  */
 286 uint16_t noisy_tx_sw_bufsz;
 287
 288 /*
 289  * Configurable value of packet buffer timeout.
 290  */
 291 uint16_t noisy_tx_sw_buf_flush_time;
 292
 293 /*
 294  * Configurable value for size of VNF internal memory area
 295  * used for simulating noisy neighbour behaviour
 296  */
 297 uint64_t noisy_lkup_mem_sz;
 298
 299 /*
 300  * Configurable value of number of random writes done in
 301  * VNF simulation memory area.
 302  */
 303 uint64_t noisy_lkup_num_writes;
 304
 305 /*
 306  * Configurable value of number of random reads done in
 307  * VNF simulation memory area.
 308  */
 309 uint64_t noisy_lkup_num_reads;
 310
 311 /*
 312  * Configurable value of number of random reads/writes done in
 313  * VNF simulation memory area.
 314  */
 315 uint64_t noisy_lkup_num_reads_writes;
 316
 317 /*
 318  * Receive Side Scaling (RSS) configuration.
 319  */
 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 321
 322 /*
 323  * Port topology configuration
 324  */
 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 326
 327 /*
 328  * Avoids to flush all the RX streams before starts forwarding.
 329  */
 330 uint8_t no_flush_rx = 0; /* flush by default */
 331
 332 /*
 333  * Flow API isolated mode.
 334  */
 335 uint8_t flow_isolate_all;
 336
 337 /*
 338  * Avoids to check link status when starting/stopping a port.
 339  */
 340 uint8_t no_link_check = 0; /* check by default */
 341
 342 /*
 343  * Don't automatically start all ports in interactive mode.
 344  */
 345 uint8_t no_device_start = 0;
 346
 347 /*
 348  * Enable link status change notification
 349  */
 350 uint8_t lsc_interrupt = 1; /* enabled by default */
 351
 352 /*
 353  * Enable device removal notification.
 354  */
 355 uint8_t rmv_interrupt = 1; /* enabled by default */
 356
 357 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 358
 359 /* After attach, port setup is called on event or by iterator */
 360 bool setup_on_probe_event = true;
 361
 362 /* Pretty printing of ethdev events */
 363 static const char * const eth_event_desc[] = {
 364         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 365         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 366         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 367         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 368         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 369         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 370         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 371         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 372         [RTE_ETH_EVENT_NEW] = "device probed",
 373         [RTE_ETH_EVENT_DESTROY] = "device released",
 374         [RTE_ETH_EVENT_MAX] = NULL,
 375 };
 376
 377 /*
 378  * Display or mask ether events
 379  * Default to all events except VF_MBOX
 380  */
 381 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 382                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 383                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 384                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 385                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 386                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 387                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 388 /*
 389  * Decide if all memory are locked for performance.
 390  */
 391 int do_mlockall = 0;
 392
 393 /*
 394  * NIC bypass mode configuration options.
 395  */
 396
 397 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 398 /* The NIC bypass watchdog timeout. */
 399 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 400 #endif
 401
 402
 403 #ifdef RTE_LIBRTE_LATENCY_STATS
 404
 405 /*
 406  * Set when latency stats is enabled in the commandline
 407  */
 408 uint8_t latencystats_enabled;
 409
 410 /*
 411  * Lcore ID to serive latency statistics.
 412  */
 413 lcoreid_t latencystats_lcore_id = -1;
 414
 415 #endif
 416
 417 /*
 418  * Ethernet device configuration.
 419  */
 420 struct rte_eth_rxmode rx_mode = {
 421         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 422                 /**< Default maximum frame length. */
 423 };
 424
 425 struct rte_eth_txmode tx_mode = {
 426         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 427 };
 428
 429 struct rte_fdir_conf fdir_conf = {
 430         .mode = RTE_FDIR_MODE_NONE,
 431         .pballoc = RTE_FDIR_PBALLOC_64K,
 432         .status = RTE_FDIR_REPORT_STATUS,
 433         .mask = {
 434                 .vlan_tci_mask = 0xFFEF,
 435                 .ipv4_mask     = {
 436                         .src_ip = 0xFFFFFFFF,
 437                         .dst_ip = 0xFFFFFFFF,
 438                 },
 439                 .ipv6_mask     = {
 440                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 441                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 442                 },
 443                 .src_port_mask = 0xFFFF,
 444                 .dst_port_mask = 0xFFFF,
 445                 .mac_addr_byte_mask = 0xFF,
 446                 .tunnel_type_mask = 1,
 447                 .tunnel_id_mask = 0xFFFFFFFF,
 448         },
 449         .drop_queue = 127,
 450 };
 451
 452 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 453
 454 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 455 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 456
 457 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 458 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 459
 460 uint16_t nb_tx_queue_stats_mappings = 0;
 461 uint16_t nb_rx_queue_stats_mappings = 0;
 462
 463 /*
 464  * Display zero values by default for xstats
 465  */
 466 uint8_t xstats_hide_zero;
 467
 468 unsigned int num_sockets = 0;
 469 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 470
 471 #ifdef RTE_LIBRTE_BITRATE
 472 /* Bitrate statistics */
 473 struct rte_stats_bitrates *bitrate_data;
 474 lcoreid_t bitrate_lcore_id;
 475 uint8_t bitrate_enabled;
 476 #endif
 477
 478 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 479 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 480
 481 struct vxlan_encap_conf vxlan_encap_conf = {
 482         .select_ipv4 = 1,
 483         .select_vlan = 0,
 484         .select_tos_ttl = 0,
 485         .vni = "\x00\x00\x00",
 486         .udp_src = 0,
 487         .udp_dst = RTE_BE16(4789),
 488         .ipv4_src = RTE_IPV4(127, 0, 0, 1),
 489         .ipv4_dst = RTE_IPV4(255, 255, 255, 255),
 490         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 491                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 492         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 493                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 494         .vlan_tci = 0,
 495         .ip_tos = 0,
 496         .ip_ttl = 255,
 497         .eth_src = "\x00\x00\x00\x00\x00\x00",
 498         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 499 };
 500
 501 struct nvgre_encap_conf nvgre_encap_conf = {
 502         .select_ipv4 = 1,
 503         .select_vlan = 0,
 504         .tni = "\x00\x00\x00",
 505         .ipv4_src = RTE_IPV4(127, 0, 0, 1),
 506         .ipv4_dst = RTE_IPV4(255, 255, 255, 255),
 507         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 508                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 509         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 510                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 511         .vlan_tci = 0,
 512         .eth_src = "\x00\x00\x00\x00\x00\x00",
 513         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 514 };
 515
 516 /* Forward function declarations */
 517 static void setup_attached_port(portid_t pi);
 518 static void map_port_queue_stats_mapping_registers(portid_t pi,
 519                                                    struct rte_port *port);
 520 static void check_all_ports_link_status(uint32_t port_mask);
 521 static int eth_event_callback(portid_t port_id,
 522                               enum rte_eth_event_type type,
 523                               void *param, void *ret_param);
 524 static void dev_event_callback(const char *device_name,
 525                                 enum rte_dev_event_type type,
 526                                 void *param);
 527
 528 /*
 529  * Check if all the ports are started.
 530  * If yes, return positive value. If not, return zero.
 531  */
 532 static int all_ports_started(void);
 533
 534 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 535 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 536
 537 /*
 538  * Helper function to check if socket is already discovered.
 539  * If yes, return positive value. If not, return zero.
 540  */
 541 int
 542 new_socket_id(unsigned int socket_id)
 543 {
 544         unsigned int i;
 545
 546         for (i = 0; i < num_sockets; i++) {
 547                 if (socket_ids[i] == socket_id)
 548                         return 0;
 549         }
 550         return 1;
 551 }
 552
 553 /*
 554  * Setup default configuration.
 555  */
 556 static void
 557 set_default_fwd_lcores_config(void)
 558 {
 559         unsigned int i;
 560         unsigned int nb_lc;
 561         unsigned int sock_num;
 562
 563         nb_lc = 0;
 564         for (i = 0; i < RTE_MAX_LCORE; i++) {
 565                 if (!rte_lcore_is_enabled(i))
 566                         continue;
 567                 sock_num = rte_lcore_to_socket_id(i);
 568                 if (new_socket_id(sock_num)) {
 569                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 570                                 rte_exit(EXIT_FAILURE,
 571                                          "Total sockets greater than %u\n",
 572                                          RTE_MAX_NUMA_NODES);
 573                         }
 574                         socket_ids[num_sockets++] = sock_num;
 575                 }
 576                 if (i == rte_get_master_lcore())
 577                         continue;
 578                 fwd_lcores_cpuids[nb_lc++] = i;
 579         }
 580         nb_lcores = (lcoreid_t) nb_lc;
 581         nb_cfg_lcores = nb_lcores;
 582         nb_fwd_lcores = 1;
 583 }
 584
 585 static void
 586 set_def_peer_eth_addrs(void)
 587 {
 588         portid_t i;
 589
 590         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 591                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 592                 peer_eth_addrs[i].addr_bytes[5] = i;
 593         }
 594 }
 595
 596 static void
 597 set_default_fwd_ports_config(void)
 598 {
 599         portid_t pt_id;
 600         int i = 0;
 601
 602         RTE_ETH_FOREACH_DEV(pt_id) {
 603                 fwd_ports_ids[i++] = pt_id;
 604
 605                 /* Update sockets info according to the attached device */
 606                 int socket_id = rte_eth_dev_socket_id(pt_id);
 607                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 608                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 609                                 rte_exit(EXIT_FAILURE,
 610                                          "Total sockets greater than %u\n",
 611                                          RTE_MAX_NUMA_NODES);
 612                         }
 613                         socket_ids[num_sockets++] = socket_id;
 614                 }
 615         }
 616
 617         nb_cfg_ports = nb_ports;
 618         nb_fwd_ports = nb_ports;
 619 }
 620
 621 void
 622 set_def_fwd_config(void)
 623 {
 624         set_default_fwd_lcores_config();
 625         set_def_peer_eth_addrs();
 626         set_default_fwd_ports_config();
 627 }
 628
 629 /* extremely pessimistic estimation of memory required to create a mempool */
 630 static int
 631 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 632 {
 633         unsigned int n_pages, mbuf_per_pg, leftover;
 634         uint64_t total_mem, mbuf_mem, obj_sz;
 635
 636         /* there is no good way to predict how much space the mempool will
 637          * occupy because it will allocate chunks on the fly, and some of those
 638          * will come from default DPDK memory while some will come from our
 639          * external memory, so just assume 128MB will be enough for everyone.
 640          */
 641         uint64_t hdr_mem = 128 << 20;
 642
 643         /* account for possible non-contiguousness */
 644         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 645         if (obj_sz > pgsz) {
 646                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 647                 return -1;
 648         }
 649
 650         mbuf_per_pg = pgsz / obj_sz;
 651         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 652         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 653
 654         mbuf_mem = n_pages * pgsz;
 655
 656         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 657
 658         if (total_mem > SIZE_MAX) {
 659                 TESTPMD_LOG(ERR, "Memory size too big\n");
 660                 return -1;
 661         }
 662         *out = (size_t)total_mem;
 663
 664         return 0;
 665 }
 666
 667 static int
 668 pagesz_flags(uint64_t page_sz)
 669 {
 670         /* as per mmap() manpage, all page sizes are log2 of page size
 671          * shifted by MAP_HUGE_SHIFT
 672          */
 673         int log2 = rte_log2_u64(page_sz);
 674
 675         return (log2 << HUGE_SHIFT);
 676 }
 677
 678 static void *
 679 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 680 {
 681         void *addr;
 682         int flags;
 683
 684         /* allocate anonymous hugepages */
 685         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 686         if (huge)
 687                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 688
 689         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 690         if (addr == MAP_FAILED)
 691                 return NULL;
 692
 693         return addr;
 694 }
 695
 696 struct extmem_param {
 697         void *addr;
 698         size_t len;
 699         size_t pgsz;
 700         rte_iova_t *iova_table;
 701         unsigned int iova_table_len;
 702 };
 703
 704 static int
 705 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 706                 bool huge)
 707 {
 708         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 709                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 710         unsigned int cur_page, n_pages, pgsz_idx;
 711         size_t mem_sz, cur_pgsz;
 712         rte_iova_t *iovas = NULL;
 713         void *addr;
 714         int ret;
 715
 716         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 717                 /* skip anything that is too big */
 718                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 719                         continue;
 720
 721                 cur_pgsz = pgsizes[pgsz_idx];
 722
 723                 /* if we were told not to allocate hugepages, override */
 724                 if (!huge)
 725                         cur_pgsz = sysconf(_SC_PAGESIZE);
 726
 727                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 728                 if (ret < 0) {
 729                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 730                         return -1;
 731                 }
 732
 733                 /* allocate our memory */
 734                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 735
 736                 /* if we couldn't allocate memory with a specified page size,
 737                  * that doesn't mean we can't do it with other page sizes, so
 738                  * try another one.
 739                  */
 740                 if (addr == NULL)
 741                         continue;
 742
 743                 /* store IOVA addresses for every page in this memory area */
 744                 n_pages = mem_sz / cur_pgsz;
 745
 746                 iovas = malloc(sizeof(*iovas) * n_pages);
 747
 748                 if (iovas == NULL) {
 749                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 750                         goto fail;
 751                 }
 752                 /* lock memory if it's not huge pages */
 753                 if (!huge)
 754                         mlock(addr, mem_sz);
 755
 756                 /* populate IOVA addresses */
 757                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 758                         rte_iova_t iova;
 759                         size_t offset;
 760                         void *cur;
 761
 762                         offset = cur_pgsz * cur_page;
 763                         cur = RTE_PTR_ADD(addr, offset);
 764
 765                         /* touch the page before getting its IOVA */
 766                         *(volatile char *)cur = 0;
 767
 768                         iova = rte_mem_virt2iova(cur);
 769
 770                         iovas[cur_page] = iova;
 771                 }
 772
 773                 break;
 774         }
 775         /* if we couldn't allocate anything */
 776         if (iovas == NULL)
 777                 return -1;
 778
 779         param->addr = addr;
 780         param->len = mem_sz;
 781         param->pgsz = cur_pgsz;
 782         param->iova_table = iovas;
 783         param->iova_table_len = n_pages;
 784
 785         return 0;
 786 fail:
 787         if (iovas)
 788                 free(iovas);
 789         if (addr)
 790                 munmap(addr, mem_sz);
 791
 792         return -1;
 793 }
 794
 795 static int
 796 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 797 {
 798         struct extmem_param param;
 799         int socket_id, ret;
 800
 801         memset(&param, 0, sizeof(param));
 802
 803         /* check if our heap exists */
 804         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 805         if (socket_id < 0) {
 806                 /* create our heap */
 807                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 808                 if (ret < 0) {
 809                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 810                         return -1;
 811                 }
 812         }
 813
 814         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 815         if (ret < 0) {
 816                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 817                 return -1;
 818         }
 819
 820         /* we now have a valid memory area, so add it to heap */
 821         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 822                         param.addr, param.len, param.iova_table,
 823                         param.iova_table_len, param.pgsz);
 824
 825         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 826
 827         /* not needed any more */
 828         free(param.iova_table);
 829
 830         if (ret < 0) {
 831                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 832                 munmap(param.addr, param.len);
 833                 return -1;
 834         }
 835
 836         /* success */
 837
 838         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 839                         param.len >> 20);
 840
 841         return 0;
 842 }
 843 static void
 844 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 845              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 846 {
 847         uint16_t pid = 0;
 848         int ret;
 849
 850         RTE_ETH_FOREACH_DEV(pid) {
 851                 struct rte_eth_dev *dev =
 852                         &rte_eth_devices[pid];
 853
 854                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 855                                         memhdr->len);
 856                 if (ret) {
 857                         TESTPMD_LOG(DEBUG,
 858                                     "unable to DMA unmap addr 0x%p "
 859                                     "for device %s\n",
 860                                     memhdr->addr, dev->data->name);
 861                 }
 862         }
 863         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 864         if (ret) {
 865                 TESTPMD_LOG(DEBUG,
 866                             "unable to un-register addr 0x%p\n", memhdr->addr);
 867         }
 868 }
 869
 870 static void
 871 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 872            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 873 {
 874         uint16_t pid = 0;
 875         size_t page_size = sysconf(_SC_PAGESIZE);
 876         int ret;
 877
 878         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 879                                   page_size);
 880         if (ret) {
 881                 TESTPMD_LOG(DEBUG,
 882                             "unable to register addr 0x%p\n", memhdr->addr);
 883                 return;
 884         }
 885         RTE_ETH_FOREACH_DEV(pid) {
 886                 struct rte_eth_dev *dev =
 887                         &rte_eth_devices[pid];
 888
 889                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 890                                       memhdr->len);
 891                 if (ret) {
 892                         TESTPMD_LOG(DEBUG,
 893                                     "unable to DMA map addr 0x%p "
 894                                     "for device %s\n",
 895                                     memhdr->addr, dev->data->name);
 896                 }
 897         }
 898 }
 899
 900 /*
 901  * Configuration initialisation done once at init time.
 902  */
 903 static struct rte_mempool *
 904 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 905                  unsigned int socket_id)
 906 {
 907         char pool_name[RTE_MEMPOOL_NAMESIZE];
 908         struct rte_mempool *rte_mp = NULL;
 909         uint32_t mb_size;
 910
 911         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 912         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 913
 914         TESTPMD_LOG(INFO,
 915                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 916                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 917
 918         switch (mp_alloc_type) {
 919         case MP_ALLOC_NATIVE:
 920                 {
 921                         /* wrapper to rte_mempool_create() */
 922                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 923                                         rte_mbuf_best_mempool_ops());
 924                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 925                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 926                         break;
 927                 }
 928         case MP_ALLOC_ANON:
 929                 {
 930                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 931                                 mb_size, (unsigned int) mb_mempool_cache,
 932                                 sizeof(struct rte_pktmbuf_pool_private),
 933                                 socket_id, mempool_flags);
 934                         if (rte_mp == NULL)
 935                                 goto err;
 936
 937                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 938                                 rte_mempool_free(rte_mp);
 939                                 rte_mp = NULL;
 940                                 goto err;
 941                         }
 942                         rte_pktmbuf_pool_init(rte_mp, NULL);
 943                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 944                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 945                         break;
 946                 }
 947         case MP_ALLOC_XMEM:
 948         case MP_ALLOC_XMEM_HUGE:
 949                 {
 950                         int heap_socket;
 951                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 952
 953                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 954                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 955
 956                         heap_socket =
 957                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 958                         if (heap_socket < 0)
 959                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 960
 961                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 962                                         rte_mbuf_best_mempool_ops());
 963                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 964                                         mb_mempool_cache, 0, mbuf_seg_size,
 965                                         heap_socket);
 966                         break;
 967                 }
 968         default:
 969                 {
 970                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 971                 }
 972         }
 973
 974 err:
 975         if (rte_mp == NULL) {
 976                 rte_exit(EXIT_FAILURE,
 977                         "Creation of mbuf pool for socket %u failed: %s\n",
 978                         socket_id, rte_strerror(rte_errno));
 979         } else if (verbose_level > 0) {
 980                 rte_mempool_dump(stdout, rte_mp);
 981         }
 982         return rte_mp;
 983 }
 984
 985 /*
 986  * Check given socket id is valid or not with NUMA mode,
 987  * if valid, return 0, else return -1
 988  */
 989 static int
 990 check_socket_id(const unsigned int socket_id)
 991 {
 992         static int warning_once = 0;
 993
 994         if (new_socket_id(socket_id)) {
 995                 if (!warning_once && numa_support)
 996                         printf("Warning: NUMA should be configured manually by"
 997                                " using --port-numa-config and"
 998                                " --ring-numa-config parameters along with"
 999                                " --numa.\n");
1000                 warning_once = 1;
1001                 return -1;
1002         }
1003         return 0;
1004 }
1005
1006 /*
1007  * Get the allowed maximum number of RX queues.
1008  * *pid return the port id which has minimal value of
1009  * max_rx_queues in all ports.
1010  */
1011 queueid_t
1012 get_allowed_max_nb_rxq(portid_t *pid)
1013 {
1014         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
1015         bool max_rxq_valid = false;
1016         portid_t pi;
1017         struct rte_eth_dev_info dev_info;
1018
1019         RTE_ETH_FOREACH_DEV(pi) {
1020                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1021                         continue;
1022
1023                 max_rxq_valid = true;
1024                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1025                         allowed_max_rxq = dev_info.max_rx_queues;
1026                         *pid = pi;
1027                 }
1028         }
1029         return max_rxq_valid ? allowed_max_rxq : 0;
1030 }
1031
1032 /*
1033  * Check input rxq is valid or not.
1034  * If input rxq is not greater than any of maximum number
1035  * of RX queues of all ports, it is valid.
1036  * if valid, return 0, else return -1
1037  */
1038 int
1039 check_nb_rxq(queueid_t rxq)
1040 {
1041         queueid_t allowed_max_rxq;
1042         portid_t pid = 0;
1043
1044         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1045         if (rxq > allowed_max_rxq) {
1046                 printf("Fail: input rxq (%u) can't be greater "
1047                        "than max_rx_queues (%u) of port %u\n",
1048                        rxq,
1049                        allowed_max_rxq,
1050                        pid);
1051                 return -1;
1052         }
1053         return 0;
1054 }
1055
1056 /*
1057  * Get the allowed maximum number of TX queues.
1058  * *pid return the port id which has minimal value of
1059  * max_tx_queues in all ports.
1060  */
1061 queueid_t
1062 get_allowed_max_nb_txq(portid_t *pid)
1063 {
1064         queueid_t allowed_max_txq = MAX_QUEUE_ID;
1065         bool max_txq_valid = false;
1066         portid_t pi;
1067         struct rte_eth_dev_info dev_info;
1068
1069         RTE_ETH_FOREACH_DEV(pi) {
1070                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1071                         continue;
1072
1073                 max_txq_valid = true;
1074                 if (dev_info.max_tx_queues < allowed_max_txq) {
1075                         allowed_max_txq = dev_info.max_tx_queues;
1076                         *pid = pi;
1077                 }
1078         }
1079         return max_txq_valid ? allowed_max_txq : 0;
1080 }
1081
1082 /*
1083  * Check input txq is valid or not.
1084  * If input txq is not greater than any of maximum number
1085  * of TX queues of all ports, it is valid.
1086  * if valid, return 0, else return -1
1087  */
1088 int
1089 check_nb_txq(queueid_t txq)
1090 {
1091         queueid_t allowed_max_txq;
1092         portid_t pid = 0;
1093
1094         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1095         if (txq > allowed_max_txq) {
1096                 printf("Fail: input txq (%u) can't be greater "
1097                        "than max_tx_queues (%u) of port %u\n",
1098                        txq,
1099                        allowed_max_txq,
1100                        pid);
1101                 return -1;
1102         }
1103         return 0;
1104 }
1105
1106 static void
1107 init_config(void)
1108 {
1109         portid_t pid;
1110         struct rte_port *port;
1111         struct rte_mempool *mbp;
1112         unsigned int nb_mbuf_per_pool;
1113         lcoreid_t  lc_id;
1114         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1115         struct rte_gro_param gro_param;
1116         uint32_t gso_types;
1117         uint16_t data_size;
1118         bool warning = 0;
1119         int k;
1120         int ret;
1121
1122         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1123
1124         /* Configuration of logical cores. */
1125         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1126                                 sizeof(struct fwd_lcore *) * nb_lcores,
1127                                 RTE_CACHE_LINE_SIZE);
1128         if (fwd_lcores == NULL) {
1129                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1130                                                         "failed\n", nb_lcores);
1131         }
1132         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1133                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1134                                                sizeof(struct fwd_lcore),
1135                                                RTE_CACHE_LINE_SIZE);
1136                 if (fwd_lcores[lc_id] == NULL) {
1137                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1138                                                                 "failed\n");
1139                 }
1140                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1141         }
1142
1143         RTE_ETH_FOREACH_DEV(pid) {
1144                 port = &ports[pid];
1145                 /* Apply default TxRx configuration for all ports */
1146                 port->dev_conf.txmode = tx_mode;
1147                 port->dev_conf.rxmode = rx_mode;
1148
1149                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1150                 if (ret != 0)
1151                         rte_exit(EXIT_FAILURE,
1152                                  "rte_eth_dev_info_get() failed\n");
1153
1154                 if (!(port->dev_info.tx_offload_capa &
1155                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1156                         port->dev_conf.txmode.offloads &=
1157                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1158                 if (!(port->dev_info.tx_offload_capa &
1159                         DEV_TX_OFFLOAD_MATCH_METADATA))
1160                         port->dev_conf.txmode.offloads &=
1161                                 ~DEV_TX_OFFLOAD_MATCH_METADATA;
1162                 if (numa_support) {
1163                         if (port_numa[pid] != NUMA_NO_CONFIG)
1164                                 port_per_socket[port_numa[pid]]++;
1165                         else {
1166                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1167
1168                                 /*
1169                                  * if socket_id is invalid,
1170                                  * set to the first available socket.
1171                                  */
1172                                 if (check_socket_id(socket_id) < 0)
1173                                         socket_id = socket_ids[0];
1174                                 port_per_socket[socket_id]++;
1175                         }
1176                 }
1177
1178                 /* Apply Rx offloads configuration */
1179                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1180                         port->rx_conf[k].offloads =
1181                                 port->dev_conf.rxmode.offloads;
1182                 /* Apply Tx offloads configuration */
1183                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1184                         port->tx_conf[k].offloads =
1185                                 port->dev_conf.txmode.offloads;
1186
1187                 /* set flag to initialize port/queue */
1188                 port->need_reconfig = 1;
1189                 port->need_reconfig_queues = 1;
1190                 port->tx_metadata = 0;
1191
1192                 /* Check for maximum number of segments per MTU. Accordingly
1193                  * update the mbuf data size.
1194                  */
1195                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1196                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1197                         data_size = rx_mode.max_rx_pkt_len /
1198                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1199
1200                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1201                                                         mbuf_data_size) {
1202                                 mbuf_data_size = data_size +
1203                                                  RTE_PKTMBUF_HEADROOM;
1204                                 warning = 1;
1205                         }
1206                 }
1207         }
1208
1209         if (warning)
1210                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1211                             mbuf_data_size);
1212
1213         /*
1214          * Create pools of mbuf.
1215          * If NUMA support is disabled, create a single pool of mbuf in
1216          * socket 0 memory by default.
1217          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1218          *
1219          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1220          * nb_txd can be configured at run time.
1221          */
1222         if (param_total_num_mbufs)
1223                 nb_mbuf_per_pool = param_total_num_mbufs;
1224         else {
1225                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1226                         (nb_lcores * mb_mempool_cache) +
1227                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1228                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1229         }
1230
1231         if (numa_support) {
1232                 uint8_t i;
1233
1234                 for (i = 0; i < num_sockets; i++)
1235                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1236                                                        nb_mbuf_per_pool,
1237                                                        socket_ids[i]);
1238         } else {
1239                 if (socket_num == UMA_NO_CONFIG)
1240                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1241                                                        nb_mbuf_per_pool, 0);
1242                 else
1243                         mempools[socket_num] = mbuf_pool_create
1244                                                         (mbuf_data_size,
1245                                                          nb_mbuf_per_pool,
1246                                                          socket_num);
1247         }
1248
1249         init_port_config();
1250
1251         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1252                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1253         /*
1254          * Records which Mbuf pool to use by each logical core, if needed.
1255          */
1256         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1257                 mbp = mbuf_pool_find(
1258                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1259
1260                 if (mbp == NULL)
1261                         mbp = mbuf_pool_find(0);
1262                 fwd_lcores[lc_id]->mbp = mbp;
1263                 /* initialize GSO context */
1264                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1265                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1266                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1267                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1268                         RTE_ETHER_CRC_LEN;
1269                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1270         }
1271
1272         /* Configuration of packet forwarding streams. */
1273         if (init_fwd_streams() < 0)
1274                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1275
1276         fwd_config_setup();
1277
1278         /* create a gro context for each lcore */
1279         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1280         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1281         gro_param.max_item_per_flow = MAX_PKT_BURST;
1282         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1283                 gro_param.socket_id = rte_lcore_to_socket_id(
1284                                 fwd_lcores_cpuids[lc_id]);
1285                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1286                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1287                         rte_exit(EXIT_FAILURE,
1288                                         "rte_gro_ctx_create() failed\n");
1289                 }
1290         }
1291
1292 #if defined RTE_LIBRTE_PMD_SOFTNIC
1293         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1294                 RTE_ETH_FOREACH_DEV(pid) {
1295                         port = &ports[pid];
1296                         const char *driver = port->dev_info.driver_name;
1297
1298                         if (strcmp(driver, "net_softnic") == 0)
1299                                 port->softport.fwd_lcore_arg = fwd_lcores;
1300                 }
1301         }
1302 #endif
1303
1304 }
1305
1306
1307 void
1308 reconfig(portid_t new_port_id, unsigned socket_id)
1309 {
1310         struct rte_port *port;
1311         int ret;
1312
1313         /* Reconfiguration of Ethernet ports. */
1314         port = &ports[new_port_id];
1315
1316         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1317         if (ret != 0)
1318                 return;
1319
1320         /* set flag to initialize port/queue */
1321         port->need_reconfig = 1;
1322         port->need_reconfig_queues = 1;
1323         port->socket_id = socket_id;
1324
1325         init_port_config();
1326 }
1327
1328
1329 int
1330 init_fwd_streams(void)
1331 {
1332         portid_t pid;
1333         struct rte_port *port;
1334         streamid_t sm_id, nb_fwd_streams_new;
1335         queueid_t q;
1336
1337         /* set socket id according to numa or not */
1338         RTE_ETH_FOREACH_DEV(pid) {
1339                 port = &ports[pid];
1340                 if (nb_rxq > port->dev_info.max_rx_queues) {
1341                         printf("Fail: nb_rxq(%d) is greater than "
1342                                 "max_rx_queues(%d)\n", nb_rxq,
1343                                 port->dev_info.max_rx_queues);
1344                         return -1;
1345                 }
1346                 if (nb_txq > port->dev_info.max_tx_queues) {
1347                         printf("Fail: nb_txq(%d) is greater than "
1348                                 "max_tx_queues(%d)\n", nb_txq,
1349                                 port->dev_info.max_tx_queues);
1350                         return -1;
1351                 }
1352                 if (numa_support) {
1353                         if (port_numa[pid] != NUMA_NO_CONFIG)
1354                                 port->socket_id = port_numa[pid];
1355                         else {
1356                                 port->socket_id = rte_eth_dev_socket_id(pid);
1357
1358                                 /*
1359                                  * if socket_id is invalid,
1360                                  * set to the first available socket.
1361                                  */
1362                                 if (check_socket_id(port->socket_id) < 0)
1363                                         port->socket_id = socket_ids[0];
1364                         }
1365                 }
1366                 else {
1367                         if (socket_num == UMA_NO_CONFIG)
1368                                 port->socket_id = 0;
1369                         else
1370                                 port->socket_id = socket_num;
1371                 }
1372         }
1373
1374         q = RTE_MAX(nb_rxq, nb_txq);
1375         if (q == 0) {
1376                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1377                 return -1;
1378         }
1379         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1380         if (nb_fwd_streams_new == nb_fwd_streams)
1381                 return 0;
1382         /* clear the old */
1383         if (fwd_streams != NULL) {
1384                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1385                         if (fwd_streams[sm_id] == NULL)
1386                                 continue;
1387                         rte_free(fwd_streams[sm_id]);
1388                         fwd_streams[sm_id] = NULL;
1389                 }
1390                 rte_free(fwd_streams);
1391                 fwd_streams = NULL;
1392         }
1393
1394         /* init new */
1395         nb_fwd_streams = nb_fwd_streams_new;
1396         if (nb_fwd_streams) {
1397                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1398                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1399                         RTE_CACHE_LINE_SIZE);
1400                 if (fwd_streams == NULL)
1401                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1402                                  " (struct fwd_stream *)) failed\n",
1403                                  nb_fwd_streams);
1404
1405                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1406                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1407                                 " struct fwd_stream", sizeof(struct fwd_stream),
1408                                 RTE_CACHE_LINE_SIZE);
1409                         if (fwd_streams[sm_id] == NULL)
1410                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1411                                          "(struct fwd_stream) failed\n");
1412                 }
1413         }
1414
1415         return 0;
1416 }
1417
1418 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1419 static void
1420 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1421 {
1422         unsigned int total_burst;
1423         unsigned int nb_burst;
1424         unsigned int burst_stats[3];
1425         uint16_t pktnb_stats[3];
1426         uint16_t nb_pkt;
1427         int burst_percent[3];
1428
1429         /*
1430          * First compute the total number of packet bursts and the
1431          * two highest numbers of bursts of the same number of packets.
1432          */
1433         total_burst = 0;
1434         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1435         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1436         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1437                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1438                 if (nb_burst == 0)
1439                         continue;
1440                 total_burst += nb_burst;
1441                 if (nb_burst > burst_stats[0]) {
1442                         burst_stats[1] = burst_stats[0];
1443                         pktnb_stats[1] = pktnb_stats[0];
1444                         burst_stats[0] = nb_burst;
1445                         pktnb_stats[0] = nb_pkt;
1446                 } else if (nb_burst > burst_stats[1]) {
1447                         burst_stats[1] = nb_burst;
1448                         pktnb_stats[1] = nb_pkt;
1449                 }
1450         }
1451         if (total_burst == 0)
1452                 return;
1453         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1454         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1455                burst_percent[0], (int) pktnb_stats[0]);
1456         if (burst_stats[0] == total_burst) {
1457                 printf("]\n");
1458                 return;
1459         }
1460         if (burst_stats[0] + burst_stats[1] == total_burst) {
1461                 printf(" + %d%% of %d pkts]\n",
1462                        100 - burst_percent[0], pktnb_stats[1]);
1463                 return;
1464         }
1465         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1466         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1467         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1468                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1469                 return;
1470         }
1471         printf(" + %d%% of %d pkts + %d%% of others]\n",
1472                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1473 }
1474 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1475
1476 static void
1477 fwd_stream_stats_display(streamid_t stream_id)
1478 {
1479         struct fwd_stream *fs;
1480         static const char *fwd_top_stats_border = "-------";
1481
1482         fs = fwd_streams[stream_id];
1483         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1484             (fs->fwd_dropped == 0))
1485                 return;
1486         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1487                "TX Port=%2d/Queue=%2d %s\n",
1488                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1489                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1490         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1491                " TX-dropped: %-14"PRIu64,
1492                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1493
1494         /* if checksum mode */
1495         if (cur_fwd_eng == &csum_fwd_engine) {
1496                 printf("  RX- bad IP checksum: %-14"PRIu64
1497                        "  Rx- bad L4 checksum: %-14"PRIu64
1498                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1499                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1500                         fs->rx_bad_outer_l4_csum);
1501         } else {
1502                 printf("\n");
1503         }
1504
1505 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1506         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1507         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1508 #endif
1509 }
1510
1511 void
1512 fwd_stats_display(void)
1513 {
1514         static const char *fwd_stats_border = "----------------------";
1515         static const char *acc_stats_border = "+++++++++++++++";
1516         struct {
1517                 struct fwd_stream *rx_stream;
1518                 struct fwd_stream *tx_stream;
1519                 uint64_t tx_dropped;
1520                 uint64_t rx_bad_ip_csum;
1521                 uint64_t rx_bad_l4_csum;
1522                 uint64_t rx_bad_outer_l4_csum;
1523         } ports_stats[RTE_MAX_ETHPORTS];
1524         uint64_t total_rx_dropped = 0;
1525         uint64_t total_tx_dropped = 0;
1526         uint64_t total_rx_nombuf = 0;
1527         struct rte_eth_stats stats;
1528 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1529         uint64_t fwd_cycles = 0;
1530 #endif
1531         uint64_t total_recv = 0;
1532         uint64_t total_xmit = 0;
1533         struct rte_port *port;
1534         streamid_t sm_id;
1535         portid_t pt_id;
1536         int i;
1537
1538         memset(ports_stats, 0, sizeof(ports_stats));
1539
1540         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1541                 struct fwd_stream *fs = fwd_streams[sm_id];
1542
1543                 if (cur_fwd_config.nb_fwd_streams >
1544                     cur_fwd_config.nb_fwd_ports) {
1545                         fwd_stream_stats_display(sm_id);
1546                 } else {
1547                         ports_stats[fs->tx_port].tx_stream = fs;
1548                         ports_stats[fs->rx_port].rx_stream = fs;
1549                 }
1550
1551                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1552
1553                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1554                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1555                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1556                                 fs->rx_bad_outer_l4_csum;
1557
1558 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1559                 fwd_cycles += fs->core_cycles;
1560 #endif
1561         }
1562         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1563                 uint8_t j;
1564
1565                 pt_id = fwd_ports_ids[i];
1566                 port = &ports[pt_id];
1567
1568                 rte_eth_stats_get(pt_id, &stats);
1569                 stats.ipackets -= port->stats.ipackets;
1570                 stats.opackets -= port->stats.opackets;
1571                 stats.ibytes -= port->stats.ibytes;
1572                 stats.obytes -= port->stats.obytes;
1573                 stats.imissed -= port->stats.imissed;
1574                 stats.oerrors -= port->stats.oerrors;
1575                 stats.rx_nombuf -= port->stats.rx_nombuf;
1576
1577                 total_recv += stats.ipackets;
1578                 total_xmit += stats.opackets;
1579                 total_rx_dropped += stats.imissed;
1580                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1581                 total_tx_dropped += stats.oerrors;
1582                 total_rx_nombuf  += stats.rx_nombuf;
1583
1584                 printf("\n  %s Forward statistics for port %-2d %s\n",
1585                        fwd_stats_border, pt_id, fwd_stats_border);
1586
1587                 if (!port->rx_queue_stats_mapping_enabled &&
1588                     !port->tx_queue_stats_mapping_enabled) {
1589                         printf("  RX-packets: %-14"PRIu64
1590                                " RX-dropped: %-14"PRIu64
1591                                "RX-total: %-"PRIu64"\n",
1592                                stats.ipackets, stats.imissed,
1593                                stats.ipackets + stats.imissed);
1594
1595                         if (cur_fwd_eng == &csum_fwd_engine)
1596                                 printf("  Bad-ipcsum: %-14"PRIu64
1597                                        " Bad-l4csum: %-14"PRIu64
1598                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1599                                        ports_stats[pt_id].rx_bad_ip_csum,
1600                                        ports_stats[pt_id].rx_bad_l4_csum,
1601                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1602                         if (stats.ierrors + stats.rx_nombuf > 0) {
1603                                 printf("  RX-error: %-"PRIu64"\n",
1604                                        stats.ierrors);
1605                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1606                                        stats.rx_nombuf);
1607                         }
1608
1609                         printf("  TX-packets: %-14"PRIu64
1610                                " TX-dropped: %-14"PRIu64
1611                                "TX-total: %-"PRIu64"\n",
1612                                stats.opackets, ports_stats[pt_id].tx_dropped,
1613                                stats.opackets + ports_stats[pt_id].tx_dropped);
1614                 } else {
1615                         printf("  RX-packets:             %14"PRIu64
1616                                "    RX-dropped:%14"PRIu64
1617                                "    RX-total:%14"PRIu64"\n",
1618                                stats.ipackets, stats.imissed,
1619                                stats.ipackets + stats.imissed);
1620
1621                         if (cur_fwd_eng == &csum_fwd_engine)
1622                                 printf("  Bad-ipcsum:%14"PRIu64
1623                                        "    Bad-l4csum:%14"PRIu64
1624                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1625                                        ports_stats[pt_id].rx_bad_ip_csum,
1626                                        ports_stats[pt_id].rx_bad_l4_csum,
1627                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1628                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1629                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1630                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1631                                        stats.rx_nombuf);
1632                         }
1633
1634                         printf("  TX-packets:             %14"PRIu64
1635                                "    TX-dropped:%14"PRIu64
1636                                "    TX-total:%14"PRIu64"\n",
1637                                stats.opackets, ports_stats[pt_id].tx_dropped,
1638                                stats.opackets + ports_stats[pt_id].tx_dropped);
1639                 }
1640
1641 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1642                 if (ports_stats[pt_id].rx_stream)
1643                         pkt_burst_stats_display("RX",
1644                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1645                 if (ports_stats[pt_id].tx_stream)
1646                         pkt_burst_stats_display("TX",
1647                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1648 #endif
1649
1650                 if (port->rx_queue_stats_mapping_enabled) {
1651                         printf("\n");
1652                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1653                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1654                                        "     RX-errors:%14"PRIu64
1655                                        "    RX-bytes:%14"PRIu64"\n",
1656                                        j, stats.q_ipackets[j],
1657                                        stats.q_errors[j], stats.q_ibytes[j]);
1658                         }
1659                         printf("\n");
1660                 }
1661                 if (port->tx_queue_stats_mapping_enabled) {
1662                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1663                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1664                                        "                                 TX-bytes:%14"
1665                                        PRIu64"\n",
1666                                        j, stats.q_opackets[j],
1667                                        stats.q_obytes[j]);
1668                         }
1669                 }
1670
1671                 printf("  %s--------------------------------%s\n",
1672                        fwd_stats_border, fwd_stats_border);
1673         }
1674
1675         printf("\n  %s Accumulated forward statistics for all ports"
1676                "%s\n",
1677                acc_stats_border, acc_stats_border);
1678         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1679                "%-"PRIu64"\n"
1680                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1681                "%-"PRIu64"\n",
1682                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1683                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1684         if (total_rx_nombuf > 0)
1685                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1686         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1687                "%s\n",
1688                acc_stats_border, acc_stats_border);
1689 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1690         if (total_recv > 0)
1691                 printf("\n  CPU cycles/packet=%u (total cycles="
1692                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1693                        (unsigned int)(fwd_cycles / total_recv),
1694                        fwd_cycles, total_recv);
1695 #endif
1696 }
1697
1698 void
1699 fwd_stats_reset(void)
1700 {
1701         streamid_t sm_id;
1702         portid_t pt_id;
1703         int i;
1704
1705         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1706                 pt_id = fwd_ports_ids[i];
1707                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1708         }
1709         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1710                 struct fwd_stream *fs = fwd_streams[sm_id];
1711
1712                 fs->rx_packets = 0;
1713                 fs->tx_packets = 0;
1714                 fs->fwd_dropped = 0;
1715                 fs->rx_bad_ip_csum = 0;
1716                 fs->rx_bad_l4_csum = 0;
1717                 fs->rx_bad_outer_l4_csum = 0;
1718
1719 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1720                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1721                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1722 #endif
1723 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1724                 fs->core_cycles = 0;
1725 #endif
1726         }
1727 }
1728
1729 static void
1730 flush_fwd_rx_queues(void)
1731 {
1732         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1733         portid_t  rxp;
1734         portid_t port_id;
1735         queueid_t rxq;
1736         uint16_t  nb_rx;
1737         uint16_t  i;
1738         uint8_t   j;
1739         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1740         uint64_t timer_period;
1741
1742         /* convert to number of cycles */
1743         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1744
1745         for (j = 0; j < 2; j++) {
1746                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1747                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1748                                 port_id = fwd_ports_ids[rxp];
1749                                 /**
1750                                 * testpmd can stuck in the below do while loop
1751                                 * if rte_eth_rx_burst() always returns nonzero
1752                                 * packets. So timer is added to exit this loop
1753                                 * after 1sec timer expiry.
1754                                 */
1755                                 prev_tsc = rte_rdtsc();
1756                                 do {
1757                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1758                                                 pkts_burst, MAX_PKT_BURST);
1759                                         for (i = 0; i < nb_rx; i++)
1760                                                 rte_pktmbuf_free(pkts_burst[i]);
1761
1762                                         cur_tsc = rte_rdtsc();
1763                                         diff_tsc = cur_tsc - prev_tsc;
1764                                         timer_tsc += diff_tsc;
1765                                 } while ((nb_rx > 0) &&
1766                                         (timer_tsc < timer_period));
1767                                 timer_tsc = 0;
1768                         }
1769                 }
1770                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1771         }
1772 }
1773
1774 static void
1775 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1776 {
1777         struct fwd_stream **fsm;
1778         streamid_t nb_fs;
1779         streamid_t sm_id;
1780 #ifdef RTE_LIBRTE_BITRATE
1781         uint64_t tics_per_1sec;
1782         uint64_t tics_datum;
1783         uint64_t tics_current;
1784         uint16_t i, cnt_ports;
1785
1786         cnt_ports = nb_ports;
1787         tics_datum = rte_rdtsc();
1788         tics_per_1sec = rte_get_timer_hz();
1789 #endif
1790         fsm = &fwd_streams[fc->stream_idx];
1791         nb_fs = fc->stream_nb;
1792         do {
1793                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1794                         (*pkt_fwd)(fsm[sm_id]);
1795 #ifdef RTE_LIBRTE_BITRATE
1796                 if (bitrate_enabled != 0 &&
1797                                 bitrate_lcore_id == rte_lcore_id()) {
1798                         tics_current = rte_rdtsc();
1799                         if (tics_current - tics_datum >= tics_per_1sec) {
1800                                 /* Periodic bitrate calculation */
1801                                 for (i = 0; i < cnt_ports; i++)
1802                                         rte_stats_bitrate_calc(bitrate_data,
1803                                                 ports_ids[i]);
1804                                 tics_datum = tics_current;
1805                         }
1806                 }
1807 #endif
1808 #ifdef RTE_LIBRTE_LATENCY_STATS
1809                 if (latencystats_enabled != 0 &&
1810                                 latencystats_lcore_id == rte_lcore_id())
1811                         rte_latencystats_update();
1812 #endif
1813
1814         } while (! fc->stopped);
1815 }
1816
1817 static int
1818 start_pkt_forward_on_core(void *fwd_arg)
1819 {
1820         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1821                              cur_fwd_config.fwd_eng->packet_fwd);
1822         return 0;
1823 }
1824
1825 /*
1826  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1827  * Used to start communication flows in network loopback test configurations.
1828  */
1829 static int
1830 run_one_txonly_burst_on_core(void *fwd_arg)
1831 {
1832         struct fwd_lcore *fwd_lc;
1833         struct fwd_lcore tmp_lcore;
1834
1835         fwd_lc = (struct fwd_lcore *) fwd_arg;
1836         tmp_lcore = *fwd_lc;
1837         tmp_lcore.stopped = 1;
1838         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1839         return 0;
1840 }
1841
1842 /*
1843  * Launch packet forwarding:
1844  *     - Setup per-port forwarding context.
1845  *     - launch logical cores with their forwarding configuration.
1846  */
1847 static void
1848 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1849 {
1850         port_fwd_begin_t port_fwd_begin;
1851         unsigned int i;
1852         unsigned int lc_id;
1853         int diag;
1854
1855         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1856         if (port_fwd_begin != NULL) {
1857                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1858                         (*port_fwd_begin)(fwd_ports_ids[i]);
1859         }
1860         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1861                 lc_id = fwd_lcores_cpuids[i];
1862                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1863                         fwd_lcores[i]->stopped = 0;
1864                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1865                                                      fwd_lcores[i], lc_id);
1866                         if (diag != 0)
1867                                 printf("launch lcore %u failed - diag=%d\n",
1868                                        lc_id, diag);
1869                 }
1870         }
1871 }
1872
1873 /*
1874  * Launch packet forwarding configuration.
1875  */
1876 void
1877 start_packet_forwarding(int with_tx_first)
1878 {
1879         port_fwd_begin_t port_fwd_begin;
1880         port_fwd_end_t  port_fwd_end;
1881         struct rte_port *port;
1882         unsigned int i;
1883         portid_t   pt_id;
1884
1885         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1886                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1887
1888         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1889                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1890
1891         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1892                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1893                 (!nb_rxq || !nb_txq))
1894                 rte_exit(EXIT_FAILURE,
1895                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1896                         cur_fwd_eng->fwd_mode_name);
1897
1898         if (all_ports_started() == 0) {
1899                 printf("Not all ports were started\n");
1900                 return;
1901         }
1902         if (test_done == 0) {
1903                 printf("Packet forwarding already started\n");
1904                 return;
1905         }
1906
1907
1908         if(dcb_test) {
1909                 for (i = 0; i < nb_fwd_ports; i++) {
1910                         pt_id = fwd_ports_ids[i];
1911                         port = &ports[pt_id];
1912                         if (!port->dcb_flag) {
1913                                 printf("In DCB mode, all forwarding ports must "
1914                                        "be configured in this mode.\n");
1915                                 return;
1916                         }
1917                 }
1918                 if (nb_fwd_lcores == 1) {
1919                         printf("In DCB mode,the nb forwarding cores "
1920                                "should be larger than 1.\n");
1921                         return;
1922                 }
1923         }
1924         test_done = 0;
1925
1926         fwd_config_setup();
1927
1928         if(!no_flush_rx)
1929                 flush_fwd_rx_queues();
1930
1931         pkt_fwd_config_display(&cur_fwd_config);
1932         rxtx_config_display();
1933
1934         fwd_stats_reset();
1935         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1936                 pt_id = fwd_ports_ids[i];
1937                 port = &ports[pt_id];
1938                 map_port_queue_stats_mapping_registers(pt_id, port);
1939         }
1940         if (with_tx_first) {
1941                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1942                 if (port_fwd_begin != NULL) {
1943                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1944                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1945                 }
1946                 while (with_tx_first--) {
1947                         launch_packet_forwarding(
1948                                         run_one_txonly_burst_on_core);
1949                         rte_eal_mp_wait_lcore();
1950                 }
1951                 port_fwd_end = tx_only_engine.port_fwd_end;
1952                 if (port_fwd_end != NULL) {
1953                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1954                                 (*port_fwd_end)(fwd_ports_ids[i]);
1955                 }
1956         }
1957         launch_packet_forwarding(start_pkt_forward_on_core);
1958 }
1959
1960 void
1961 stop_packet_forwarding(void)
1962 {
1963         port_fwd_end_t port_fwd_end;
1964         lcoreid_t lc_id;
1965         portid_t pt_id;
1966         int i;
1967
1968         if (test_done) {
1969                 printf("Packet forwarding not started\n");
1970                 return;
1971         }
1972         printf("Telling cores to stop...");
1973         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1974                 fwd_lcores[lc_id]->stopped = 1;
1975         printf("\nWaiting for lcores to finish...\n");
1976         rte_eal_mp_wait_lcore();
1977         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1978         if (port_fwd_end != NULL) {
1979                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1980                         pt_id = fwd_ports_ids[i];
1981                         (*port_fwd_end)(pt_id);
1982                 }
1983         }
1984
1985         fwd_stats_display();
1986
1987         printf("\nDone.\n");
1988         test_done = 1;
1989 }
1990
1991 void
1992 dev_set_link_up(portid_t pid)
1993 {
1994         if (rte_eth_dev_set_link_up(pid) < 0)
1995                 printf("\nSet link up fail.\n");
1996 }
1997
1998 void
1999 dev_set_link_down(portid_t pid)
2000 {
2001         if (rte_eth_dev_set_link_down(pid) < 0)
2002                 printf("\nSet link down fail.\n");
2003 }
2004
2005 static int
2006 all_ports_started(void)
2007 {
2008         portid_t pi;
2009         struct rte_port *port;
2010
2011         RTE_ETH_FOREACH_DEV(pi) {
2012                 port = &ports[pi];
2013                 /* Check if there is a port which is not started */
2014                 if ((port->port_status != RTE_PORT_STARTED) &&
2015                         (port->slave_flag == 0))
2016                         return 0;
2017         }
2018
2019         /* No port is not started */
2020         return 1;
2021 }
2022
2023 int
2024 port_is_stopped(portid_t port_id)
2025 {
2026         struct rte_port *port = &ports[port_id];
2027
2028         if ((port->port_status != RTE_PORT_STOPPED) &&
2029             (port->slave_flag == 0))
2030                 return 0;
2031         return 1;
2032 }
2033
2034 int
2035 all_ports_stopped(void)
2036 {
2037         portid_t pi;
2038
2039         RTE_ETH_FOREACH_DEV(pi) {
2040                 if (!port_is_stopped(pi))
2041                         return 0;
2042         }
2043
2044         return 1;
2045 }
2046
2047 int
2048 port_is_started(portid_t port_id)
2049 {
2050         if (port_id_is_invalid(port_id, ENABLED_WARN))
2051                 return 0;
2052
2053         if (ports[port_id].port_status != RTE_PORT_STARTED)
2054                 return 0;
2055
2056         return 1;
2057 }
2058
2059 int
2060 start_port(portid_t pid)
2061 {
2062         int diag, need_check_link_status = -1;
2063         portid_t pi;
2064         queueid_t qi;
2065         struct rte_port *port;
2066         struct rte_ether_addr mac_addr;
2067
2068         if (port_id_is_invalid(pid, ENABLED_WARN))
2069                 return 0;
2070
2071         if(dcb_config)
2072                 dcb_test = 1;
2073         RTE_ETH_FOREACH_DEV(pi) {
2074                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2075                         continue;
2076
2077                 need_check_link_status = 0;
2078                 port = &ports[pi];
2079                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2080                                                  RTE_PORT_HANDLING) == 0) {
2081                         printf("Port %d is now not stopped\n", pi);
2082                         continue;
2083                 }
2084
2085                 if (port->need_reconfig > 0) {
2086                         port->need_reconfig = 0;
2087
2088                         if (flow_isolate_all) {
2089                                 int ret = port_flow_isolate(pi, 1);
2090                                 if (ret) {
2091                                         printf("Failed to apply isolated"
2092                                                " mode on port %d\n", pi);
2093                                         return -1;
2094                                 }
2095                         }
2096                         configure_rxtx_dump_callbacks(0);
2097                         printf("Configuring Port %d (socket %u)\n", pi,
2098                                         port->socket_id);
2099                         /* configure port */
2100                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
2101                                                 &(port->dev_conf));
2102                         if (diag != 0) {
2103                                 if (rte_atomic16_cmpset(&(port->port_status),
2104                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2105                                         printf("Port %d can not be set back "
2106                                                         "to stopped\n", pi);
2107                                 printf("Fail to configure port %d\n", pi);
2108                                 /* try to reconfigure port next time */
2109                                 port->need_reconfig = 1;
2110                                 return -1;
2111                         }
2112                 }
2113                 if (port->need_reconfig_queues > 0) {
2114                         port->need_reconfig_queues = 0;
2115                         /* setup tx queues */
2116                         for (qi = 0; qi < nb_txq; qi++) {
2117                                 if ((numa_support) &&
2118                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2119                                         diag = rte_eth_tx_queue_setup(pi, qi,
2120                                                 port->nb_tx_desc[qi],
2121                                                 txring_numa[pi],
2122                                                 &(port->tx_conf[qi]));
2123                                 else
2124                                         diag = rte_eth_tx_queue_setup(pi, qi,
2125                                                 port->nb_tx_desc[qi],
2126                                                 port->socket_id,
2127                                                 &(port->tx_conf[qi]));
2128
2129                                 if (diag == 0)
2130                                         continue;
2131
2132                                 /* Fail to setup tx queue, return */
2133                                 if (rte_atomic16_cmpset(&(port->port_status),
2134                                                         RTE_PORT_HANDLING,
2135                                                         RTE_PORT_STOPPED) == 0)
2136                                         printf("Port %d can not be set back "
2137                                                         "to stopped\n", pi);
2138                                 printf("Fail to configure port %d tx queues\n",
2139                                        pi);
2140                                 /* try to reconfigure queues next time */
2141                                 port->need_reconfig_queues = 1;
2142                                 return -1;
2143                         }
2144                         for (qi = 0; qi < nb_rxq; qi++) {
2145                                 /* setup rx queues */
2146                                 if ((numa_support) &&
2147                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2148                                         struct rte_mempool * mp =
2149                                                 mbuf_pool_find(rxring_numa[pi]);
2150                                         if (mp == NULL) {
2151                                                 printf("Failed to setup RX queue:"
2152                                                         "No mempool allocation"
2153                                                         " on the socket %d\n",
2154                                                         rxring_numa[pi]);
2155                                                 return -1;
2156                                         }
2157
2158                                         diag = rte_eth_rx_queue_setup(pi, qi,
2159                                              port->nb_rx_desc[qi],
2160                                              rxring_numa[pi],
2161                                              &(port->rx_conf[qi]),
2162                                              mp);
2163                                 } else {
2164                                         struct rte_mempool *mp =
2165                                                 mbuf_pool_find(port->socket_id);
2166                                         if (mp == NULL) {
2167                                                 printf("Failed to setup RX queue:"
2168                                                         "No mempool allocation"
2169                                                         " on the socket %d\n",
2170                                                         port->socket_id);
2171                                                 return -1;
2172                                         }
2173                                         diag = rte_eth_rx_queue_setup(pi, qi,
2174                                              port->nb_rx_desc[qi],
2175                                              port->socket_id,
2176                                              &(port->rx_conf[qi]),
2177                                              mp);
2178                                 }
2179                                 if (diag == 0)
2180                                         continue;
2181
2182                                 /* Fail to setup rx queue, return */
2183                                 if (rte_atomic16_cmpset(&(port->port_status),
2184                                                         RTE_PORT_HANDLING,
2185                                                         RTE_PORT_STOPPED) == 0)
2186                                         printf("Port %d can not be set back "
2187                                                         "to stopped\n", pi);
2188                                 printf("Fail to configure port %d rx queues\n",
2189                                        pi);
2190                                 /* try to reconfigure queues next time */
2191                                 port->need_reconfig_queues = 1;
2192                                 return -1;
2193                         }
2194                 }
2195                 configure_rxtx_dump_callbacks(verbose_level);
2196                 /* start port */
2197                 if (rte_eth_dev_start(pi) < 0) {
2198                         printf("Fail to start port %d\n", pi);
2199
2200                         /* Fail to setup rx queue, return */
2201                         if (rte_atomic16_cmpset(&(port->port_status),
2202                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2203                                 printf("Port %d can not be set back to "
2204                                                         "stopped\n", pi);
2205                         continue;
2206                 }
2207
2208                 if (rte_atomic16_cmpset(&(port->port_status),
2209                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2210                         printf("Port %d can not be set into started\n", pi);
2211
2212                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2213                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2214                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2215                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2216                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2217
2218                 /* at least one port started, need checking link status */
2219                 need_check_link_status = 1;
2220         }
2221
2222         if (need_check_link_status == 1 && !no_link_check)
2223                 check_all_ports_link_status(RTE_PORT_ALL);
2224         else if (need_check_link_status == 0)
2225                 printf("Please stop the ports first\n");
2226
2227         printf("Done\n");
2228         return 0;
2229 }
2230
2231 void
2232 stop_port(portid_t pid)
2233 {
2234         portid_t pi;
2235         struct rte_port *port;
2236         int need_check_link_status = 0;
2237
2238         if (dcb_test) {
2239                 dcb_test = 0;
2240                 dcb_config = 0;
2241         }
2242
2243         if (port_id_is_invalid(pid, ENABLED_WARN))
2244                 return;
2245
2246         printf("Stopping ports...\n");
2247
2248         RTE_ETH_FOREACH_DEV(pi) {
2249                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2250                         continue;
2251
2252                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2253                         printf("Please remove port %d from forwarding configuration.\n", pi);
2254                         continue;
2255                 }
2256
2257                 if (port_is_bonding_slave(pi)) {
2258                         printf("Please remove port %d from bonded device.\n", pi);
2259                         continue;
2260                 }
2261
2262                 port = &ports[pi];
2263                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2264                                                 RTE_PORT_HANDLING) == 0)
2265                         continue;
2266
2267                 rte_eth_dev_stop(pi);
2268
2269                 if (rte_atomic16_cmpset(&(port->port_status),
2270                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2271                         printf("Port %d can not be set into stopped\n", pi);
2272                 need_check_link_status = 1;
2273         }
2274         if (need_check_link_status && !no_link_check)
2275                 check_all_ports_link_status(RTE_PORT_ALL);
2276
2277         printf("Done\n");
2278 }
2279
2280 static void
2281 remove_invalid_ports_in(portid_t *array, portid_t *total)
2282 {
2283         portid_t i;
2284         portid_t new_total = 0;
2285
2286         for (i = 0; i < *total; i++)
2287                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2288                         array[new_total] = array[i];
2289                         new_total++;
2290                 }
2291         *total = new_total;
2292 }
2293
2294 static void
2295 remove_invalid_ports(void)
2296 {
2297         remove_invalid_ports_in(ports_ids, &nb_ports);
2298         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2299         nb_cfg_ports = nb_fwd_ports;
2300 }
2301
2302 void
2303 close_port(portid_t pid)
2304 {
2305         portid_t pi;
2306         struct rte_port *port;
2307
2308         if (port_id_is_invalid(pid, ENABLED_WARN))
2309                 return;
2310
2311         printf("Closing ports...\n");
2312
2313         RTE_ETH_FOREACH_DEV(pi) {
2314                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2315                         continue;
2316
2317                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2318                         printf("Please remove port %d from forwarding configuration.\n", pi);
2319                         continue;
2320                 }
2321
2322                 if (port_is_bonding_slave(pi)) {
2323                         printf("Please remove port %d from bonded device.\n", pi);
2324                         continue;
2325                 }
2326
2327                 port = &ports[pi];
2328                 if (rte_atomic16_cmpset(&(port->port_status),
2329                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2330                         printf("Port %d is already closed\n", pi);
2331                         continue;
2332                 }
2333
2334                 if (rte_atomic16_cmpset(&(port->port_status),
2335                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2336                         printf("Port %d is now not stopped\n", pi);
2337                         continue;
2338                 }
2339
2340                 if (port->flow_list)
2341                         port_flow_flush(pi);
2342                 rte_eth_dev_close(pi);
2343
2344                 remove_invalid_ports();
2345
2346                 if (rte_atomic16_cmpset(&(port->port_status),
2347                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2348                         printf("Port %d cannot be set to closed\n", pi);
2349         }
2350
2351         printf("Done\n");
2352 }
2353
2354 void
2355 reset_port(portid_t pid)
2356 {
2357         int diag;
2358         portid_t pi;
2359         struct rte_port *port;
2360
2361         if (port_id_is_invalid(pid, ENABLED_WARN))
2362                 return;
2363
2364         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2365                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2366                 printf("Can not reset port(s), please stop port(s) first.\n");
2367                 return;
2368         }
2369
2370         printf("Resetting ports...\n");
2371
2372         RTE_ETH_FOREACH_DEV(pi) {
2373                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2374                         continue;
2375
2376                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2377                         printf("Please remove port %d from forwarding "
2378                                "configuration.\n", pi);
2379                         continue;
2380                 }
2381
2382                 if (port_is_bonding_slave(pi)) {
2383                         printf("Please remove port %d from bonded device.\n",
2384                                pi);
2385                         continue;
2386                 }
2387
2388                 diag = rte_eth_dev_reset(pi);
2389                 if (diag == 0) {
2390                         port = &ports[pi];
2391                         port->need_reconfig = 1;
2392                         port->need_reconfig_queues = 1;
2393                 } else {
2394                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2395                 }
2396         }
2397
2398         printf("Done\n");
2399 }
2400
2401 void
2402 attach_port(char *identifier)
2403 {
2404         portid_t pi;
2405         struct rte_dev_iterator iterator;
2406
2407         printf("Attaching a new port...\n");
2408
2409         if (identifier == NULL) {
2410                 printf("Invalid parameters are specified\n");
2411                 return;
2412         }
2413
2414         if (rte_dev_probe(identifier) < 0) {
2415                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2416                 return;
2417         }
2418
2419         /* first attach mode: event */
2420         if (setup_on_probe_event) {
2421                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2422                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2423                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2424                                         ports[pi].need_setup != 0)
2425                                 setup_attached_port(pi);
2426                 return;
2427         }
2428
2429         /* second attach mode: iterator */
2430         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2431                 /* setup ports matching the devargs used for probing */
2432                 if (port_is_forwarding(pi))
2433                         continue; /* port was already attached before */
2434                 setup_attached_port(pi);
2435         }
2436 }
2437
2438 static void
2439 setup_attached_port(portid_t pi)
2440 {
2441         unsigned int socket_id;
2442         int ret;
2443
2444         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2445         /* if socket_id is invalid, set to the first available socket. */
2446         if (check_socket_id(socket_id) < 0)
2447                 socket_id = socket_ids[0];
2448         reconfig(pi, socket_id);
2449         ret = rte_eth_promiscuous_enable(pi);
2450         if (ret != 0)
2451                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2452                         pi, rte_strerror(-ret));
2453
2454         ports_ids[nb_ports++] = pi;
2455         fwd_ports_ids[nb_fwd_ports++] = pi;
2456         nb_cfg_ports = nb_fwd_ports;
2457         ports[pi].need_setup = 0;
2458         ports[pi].port_status = RTE_PORT_STOPPED;
2459
2460         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2461         printf("Done\n");
2462 }
2463
2464 void
2465 detach_port_device(portid_t port_id)
2466 {
2467         struct rte_device *dev;
2468         portid_t sibling;
2469
2470         printf("Removing a device...\n");
2471
2472         dev = rte_eth_devices[port_id].device;
2473         if (dev == NULL) {
2474                 printf("Device already removed\n");
2475                 return;
2476         }
2477
2478         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2479                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2480                         printf("Port not stopped\n");
2481                         return;
2482                 }
2483                 printf("Port was not closed\n");
2484                 if (ports[port_id].flow_list)
2485                         port_flow_flush(port_id);
2486         }
2487
2488         if (rte_dev_remove(dev) < 0) {
2489                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2490                 return;
2491         }
2492         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2493                 /* reset mapping between old ports and removed device */
2494                 rte_eth_devices[sibling].device = NULL;
2495                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2496                         /* sibling ports are forced to be closed */
2497                         ports[sibling].port_status = RTE_PORT_CLOSED;
2498                         printf("Port %u is closed\n", sibling);
2499                 }
2500         }
2501
2502         remove_invalid_ports();
2503
2504         printf("Device of port %u is detached\n", port_id);
2505         printf("Now total ports is %d\n", nb_ports);
2506         printf("Done\n");
2507         return;
2508 }
2509
2510 void
2511 detach_device(char *identifier)
2512 {
2513         struct rte_dev_iterator iterator;
2514         struct rte_devargs da;
2515         portid_t port_id;
2516
2517         printf("Removing a device...\n");
2518
2519         memset(&da, 0, sizeof(da));
2520         if (rte_devargs_parsef(&da, "%s", identifier)) {
2521                 printf("cannot parse identifier\n");
2522                 if (da.args)
2523                         free(da.args);
2524                 return;
2525         }
2526
2527         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2528                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2529                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2530                                 printf("Port %u not stopped\n", port_id);
2531                                 return;
2532                         }
2533
2534                         /* sibling ports are forced to be closed */
2535                         if (ports[port_id].flow_list)
2536                                 port_flow_flush(port_id);
2537                         ports[port_id].port_status = RTE_PORT_CLOSED;
2538                         printf("Port %u is now closed\n", port_id);
2539                 }
2540         }
2541
2542         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2543                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2544                             da.name, da.bus->name);
2545                 return;
2546         }
2547
2548         remove_invalid_ports();
2549
2550         printf("Device %s is detached\n", identifier);
2551         printf("Now total ports is %d\n", nb_ports);
2552         printf("Done\n");
2553 }
2554
2555 void
2556 pmd_test_exit(void)
2557 {
2558         portid_t pt_id;
2559         int ret;
2560         int i;
2561
2562         if (test_done == 0)
2563                 stop_packet_forwarding();
2564
2565         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2566                 if (mempools[i]) {
2567                         if (mp_alloc_type == MP_ALLOC_ANON)
2568                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2569                                                      NULL);
2570                 }
2571         }
2572         if (ports != NULL) {
2573                 no_link_check = 1;
2574                 RTE_ETH_FOREACH_DEV(pt_id) {
2575                         printf("\nStopping port %d...\n", pt_id);
2576                         fflush(stdout);
2577                         stop_port(pt_id);
2578                 }
2579                 RTE_ETH_FOREACH_DEV(pt_id) {
2580                         printf("\nShutting down port %d...\n", pt_id);
2581                         fflush(stdout);
2582                         close_port(pt_id);
2583                 }
2584         }
2585
2586         if (hot_plug) {
2587                 ret = rte_dev_event_monitor_stop();
2588                 if (ret) {
2589                         RTE_LOG(ERR, EAL,
2590                                 "fail to stop device event monitor.");
2591                         return;
2592                 }
2593
2594                 ret = rte_dev_event_callback_unregister(NULL,
2595                         dev_event_callback, NULL);
2596                 if (ret < 0) {
2597                         RTE_LOG(ERR, EAL,
2598                                 "fail to unregister device event callback.\n");
2599                         return;
2600                 }
2601
2602                 ret = rte_dev_hotplug_handle_disable();
2603                 if (ret) {
2604                         RTE_LOG(ERR, EAL,
2605                                 "fail to disable hotplug handling.\n");
2606                         return;
2607                 }
2608         }
2609         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2610                 if (mempools[i])
2611                         rte_mempool_free(mempools[i]);
2612         }
2613
2614         printf("\nBye...\n");
2615 }
2616
2617 typedef void (*cmd_func_t)(void);
2618 struct pmd_test_command {
2619         const char *cmd_name;
2620         cmd_func_t cmd_func;
2621 };
2622
2623 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2624
2625 /* Check the link status of all ports in up to 9s, and print them finally */
2626 static void
2627 check_all_ports_link_status(uint32_t port_mask)
2628 {
2629 #define CHECK_INTERVAL 100 /* 100ms */
2630 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2631         portid_t portid;
2632         uint8_t count, all_ports_up, print_flag = 0;
2633         struct rte_eth_link link;
2634         int ret;
2635
2636         printf("Checking link statuses...\n");
2637         fflush(stdout);
2638         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2639                 all_ports_up = 1;
2640                 RTE_ETH_FOREACH_DEV(portid) {
2641                         if ((port_mask & (1 << portid)) == 0)
2642                                 continue;
2643                         memset(&link, 0, sizeof(link));
2644                         ret = rte_eth_link_get_nowait(portid, &link);
2645                         if (ret < 0) {
2646                                 all_ports_up = 0;
2647                                 if (print_flag == 1)
2648                                         printf("Port %u link get failed: %s\n",
2649                                                 portid, rte_strerror(-ret));
2650                                 continue;
2651                         }
2652                         /* print link status if flag set */
2653                         if (print_flag == 1) {
2654                                 if (link.link_status)
2655                                         printf(
2656                                         "Port%d Link Up. speed %u Mbps- %s\n",
2657                                         portid, link.link_speed,
2658                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2659                                         ("full-duplex") : ("half-duplex\n"));
2660                                 else
2661                                         printf("Port %d Link Down\n", portid);
2662                                 continue;
2663                         }
2664                         /* clear all_ports_up flag if any link down */
2665                         if (link.link_status == ETH_LINK_DOWN) {
2666                                 all_ports_up = 0;
2667                                 break;
2668                         }
2669                 }
2670                 /* after finally printing all link status, get out */
2671                 if (print_flag == 1)
2672                         break;
2673
2674                 if (all_ports_up == 0) {
2675                         fflush(stdout);
2676                         rte_delay_ms(CHECK_INTERVAL);
2677                 }
2678
2679                 /* set the print_flag if all ports up or timeout */
2680                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2681                         print_flag = 1;
2682                 }
2683
2684                 if (lsc_interrupt)
2685                         break;
2686         }
2687 }
2688
2689 /*
2690  * This callback is for remove a port for a device. It has limitation because
2691  * it is not for multiple port removal for a device.
2692  * TODO: the device detach invoke will plan to be removed from user side to
2693  * eal. And convert all PMDs to free port resources on ether device closing.
2694  */
2695 static void
2696 rmv_port_callback(void *arg)
2697 {
2698         int need_to_start = 0;
2699         int org_no_link_check = no_link_check;
2700         portid_t port_id = (intptr_t)arg;
2701
2702         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2703
2704         if (!test_done && port_is_forwarding(port_id)) {
2705                 need_to_start = 1;
2706                 stop_packet_forwarding();
2707         }
2708         no_link_check = 1;
2709         stop_port(port_id);
2710         no_link_check = org_no_link_check;
2711         close_port(port_id);
2712         detach_port_device(port_id);
2713         if (need_to_start)
2714                 start_packet_forwarding(0);
2715 }
2716
2717 /* This function is used by the interrupt thread */
2718 static int
2719 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2720                   void *ret_param)
2721 {
2722         RTE_SET_USED(param);
2723         RTE_SET_USED(ret_param);
2724
2725         if (type >= RTE_ETH_EVENT_MAX) {
2726                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2727                         port_id, __func__, type);
2728                 fflush(stderr);
2729         } else if (event_print_mask & (UINT32_C(1) << type)) {
2730                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2731                         eth_event_desc[type]);
2732                 fflush(stdout);
2733         }
2734
2735         switch (type) {
2736         case RTE_ETH_EVENT_NEW:
2737                 ports[port_id].need_setup = 1;
2738                 ports[port_id].port_status = RTE_PORT_HANDLING;
2739                 break;
2740         case RTE_ETH_EVENT_INTR_RMV:
2741                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2742                         break;
2743                 if (rte_eal_alarm_set(100000,
2744                                 rmv_port_callback, (void *)(intptr_t)port_id))
2745                         fprintf(stderr, "Could not set up deferred device removal\n");
2746                 break;
2747         default:
2748                 break;
2749         }
2750         return 0;
2751 }
2752
2753 static int
2754 register_eth_event_callback(void)
2755 {
2756         int ret;
2757         enum rte_eth_event_type event;
2758
2759         for (event = RTE_ETH_EVENT_UNKNOWN;
2760                         event < RTE_ETH_EVENT_MAX; event++) {
2761                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2762                                 event,
2763                                 eth_event_callback,
2764                                 NULL);
2765                 if (ret != 0) {
2766                         TESTPMD_LOG(ERR, "Failed to register callback for "
2767                                         "%s event\n", eth_event_desc[event]);
2768                         return -1;
2769                 }
2770         }
2771
2772         return 0;
2773 }
2774
2775 /* This function is used by the interrupt thread */
2776 static void
2777 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2778                              __rte_unused void *arg)
2779 {
2780         uint16_t port_id;
2781         int ret;
2782
2783         if (type >= RTE_DEV_EVENT_MAX) {
2784                 fprintf(stderr, "%s called upon invalid event %d\n",
2785                         __func__, type);
2786                 fflush(stderr);
2787         }
2788
2789         switch (type) {
2790         case RTE_DEV_EVENT_REMOVE:
2791                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2792                         device_name);
2793                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2794                 if (ret) {
2795                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2796                                 device_name);
2797                         return;
2798                 }
2799                 /*
2800                  * Because the user's callback is invoked in eal interrupt
2801                  * callback, the interrupt callback need to be finished before
2802                  * it can be unregistered when detaching device. So finish
2803                  * callback soon and use a deferred removal to detach device
2804                  * is need. It is a workaround, once the device detaching be
2805                  * moved into the eal in the future, the deferred removal could
2806                  * be deleted.
2807                  */
2808                 if (rte_eal_alarm_set(100000,
2809                                 rmv_port_callback, (void *)(intptr_t)port_id))
2810                         RTE_LOG(ERR, EAL,
2811                                 "Could not set up deferred device removal\n");
2812                 break;
2813         case RTE_DEV_EVENT_ADD:
2814                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2815                         device_name);
2816                 /* TODO: After finish kernel driver binding,
2817                  * begin to attach port.
2818                  */
2819                 break;
2820         default:
2821                 break;
2822         }
2823 }
2824
2825 static int
2826 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2827 {
2828         uint16_t i;
2829         int diag;
2830         uint8_t mapping_found = 0;
2831
2832         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2833                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2834                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2835                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2836                                         tx_queue_stats_mappings[i].queue_id,
2837                                         tx_queue_stats_mappings[i].stats_counter_id);
2838                         if (diag != 0)
2839                                 return diag;
2840                         mapping_found = 1;
2841                 }
2842         }
2843         if (mapping_found)
2844                 port->tx_queue_stats_mapping_enabled = 1;
2845         return 0;
2846 }
2847
2848 static int
2849 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2850 {
2851         uint16_t i;
2852         int diag;
2853         uint8_t mapping_found = 0;
2854
2855         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2856                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2857                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2858                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2859                                         rx_queue_stats_mappings[i].queue_id,
2860                                         rx_queue_stats_mappings[i].stats_counter_id);
2861                         if (diag != 0)
2862                                 return diag;
2863                         mapping_found = 1;
2864                 }
2865         }
2866         if (mapping_found)
2867                 port->rx_queue_stats_mapping_enabled = 1;
2868         return 0;
2869 }
2870
2871 static void
2872 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2873 {
2874         int diag = 0;
2875
2876         diag = set_tx_queue_stats_mapping_registers(pi, port);
2877         if (diag != 0) {
2878                 if (diag == -ENOTSUP) {
2879                         port->tx_queue_stats_mapping_enabled = 0;
2880                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2881                 }
2882                 else
2883                         rte_exit(EXIT_FAILURE,
2884                                         "set_tx_queue_stats_mapping_registers "
2885                                         "failed for port id=%d diag=%d\n",
2886                                         pi, diag);
2887         }
2888
2889         diag = set_rx_queue_stats_mapping_registers(pi, port);
2890         if (diag != 0) {
2891                 if (diag == -ENOTSUP) {
2892                         port->rx_queue_stats_mapping_enabled = 0;
2893                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2894                 }
2895                 else
2896                         rte_exit(EXIT_FAILURE,
2897                                         "set_rx_queue_stats_mapping_registers "
2898                                         "failed for port id=%d diag=%d\n",
2899                                         pi, diag);
2900         }
2901 }
2902
2903 static void
2904 rxtx_port_config(struct rte_port *port)
2905 {
2906         uint16_t qid;
2907         uint64_t offloads;
2908
2909         for (qid = 0; qid < nb_rxq; qid++) {
2910                 offloads = port->rx_conf[qid].offloads;
2911                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2912                 if (offloads != 0)
2913                         port->rx_conf[qid].offloads = offloads;
2914
2915                 /* Check if any Rx parameters have been passed */
2916                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2917                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2918
2919                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2920                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2921
2922                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2923                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2924
2925                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2926                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2927
2928                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2929                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2930
2931                 port->nb_rx_desc[qid] = nb_rxd;
2932         }
2933
2934         for (qid = 0; qid < nb_txq; qid++) {
2935                 offloads = port->tx_conf[qid].offloads;
2936                 port->tx_conf[qid] = port->dev_info.default_txconf;
2937                 if (offloads != 0)
2938                         port->tx_conf[qid].offloads = offloads;
2939
2940                 /* Check if any Tx parameters have been passed */
2941                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2942                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2943
2944                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2945                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2946
2947                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2948                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2949
2950                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2951                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2952
2953                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2954                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2955
2956                 port->nb_tx_desc[qid] = nb_txd;
2957         }
2958 }
2959
2960 void
2961 init_port_config(void)
2962 {
2963         portid_t pid;
2964         struct rte_port *port;
2965         int ret;
2966
2967         RTE_ETH_FOREACH_DEV(pid) {
2968                 port = &ports[pid];
2969                 port->dev_conf.fdir_conf = fdir_conf;
2970
2971                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
2972                 if (ret != 0)
2973                         return;
2974
2975                 if (nb_rxq > 1) {
2976                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2977                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2978                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2979                 } else {
2980                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2981                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2982                 }
2983
2984                 if (port->dcb_flag == 0) {
2985                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2986                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2987                         else
2988                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2989                 }
2990
2991                 rxtx_port_config(port);
2992
2993                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
2994                 if (ret != 0)
2995                         return;
2996
2997                 map_port_queue_stats_mapping_registers(pid, port);
2998 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2999                 rte_pmd_ixgbe_bypass_init(pid);
3000 #endif
3001
3002                 if (lsc_interrupt &&
3003                     (rte_eth_devices[pid].data->dev_flags &
3004                      RTE_ETH_DEV_INTR_LSC))
3005                         port->dev_conf.intr_conf.lsc = 1;
3006                 if (rmv_interrupt &&
3007                     (rte_eth_devices[pid].data->dev_flags &
3008                      RTE_ETH_DEV_INTR_RMV))
3009                         port->dev_conf.intr_conf.rmv = 1;
3010         }
3011 }
3012
3013 void set_port_slave_flag(portid_t slave_pid)
3014 {
3015         struct rte_port *port;
3016
3017         port = &ports[slave_pid];
3018         port->slave_flag = 1;
3019 }
3020
3021 void clear_port_slave_flag(portid_t slave_pid)
3022 {
3023         struct rte_port *port;
3024
3025         port = &ports[slave_pid];
3026         port->slave_flag = 0;
3027 }
3028
3029 uint8_t port_is_bonding_slave(portid_t slave_pid)
3030 {
3031         struct rte_port *port;
3032
3033         port = &ports[slave_pid];
3034         if ((rte_eth_devices[slave_pid].data->dev_flags &
3035             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3036                 return 1;
3037         return 0;
3038 }
3039
3040 const uint16_t vlan_tags[] = {
3041                 0,  1,  2,  3,  4,  5,  6,  7,
3042                 8,  9, 10, 11,  12, 13, 14, 15,
3043                 16, 17, 18, 19, 20, 21, 22, 23,
3044                 24, 25, 26, 27, 28, 29, 30, 31
3045 };
3046
3047 static  int
3048 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3049                  enum dcb_mode_enable dcb_mode,
3050                  enum rte_eth_nb_tcs num_tcs,
3051                  uint8_t pfc_en)
3052 {
3053         uint8_t i;
3054         int32_t rc;
3055         struct rte_eth_rss_conf rss_conf;
3056
3057         /*
3058          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3059          * given above, and the number of traffic classes available for use.
3060          */
3061         if (dcb_mode == DCB_VT_ENABLED) {
3062                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3063                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3064                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3065                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3066
3067                 /* VMDQ+DCB RX and TX configurations */
3068                 vmdq_rx_conf->enable_default_pool = 0;
3069                 vmdq_rx_conf->default_pool = 0;
3070                 vmdq_rx_conf->nb_queue_pools =
3071                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3072                 vmdq_tx_conf->nb_queue_pools =
3073                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3074
3075                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3076                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3077                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3078                         vmdq_rx_conf->pool_map[i].pools =
3079                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3080                 }
3081                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3082                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3083                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3084                 }
3085
3086                 /* set DCB mode of RX and TX of multiple queues */
3087                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
3088                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3089         } else {
3090                 struct rte_eth_dcb_rx_conf *rx_conf =
3091                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3092                 struct rte_eth_dcb_tx_conf *tx_conf =
3093                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3094
3095                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3096                 if (rc != 0)
3097                         return rc;
3098
3099                 rx_conf->nb_tcs = num_tcs;
3100                 tx_conf->nb_tcs = num_tcs;
3101
3102                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3103                         rx_conf->dcb_tc[i] = i % num_tcs;
3104                         tx_conf->dcb_tc[i] = i % num_tcs;
3105                 }
3106
3107                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
3108                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3109                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3110         }
3111
3112         if (pfc_en)
3113                 eth_conf->dcb_capability_en =
3114                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3115         else
3116                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3117
3118         return 0;
3119 }
3120
3121 int
3122 init_port_dcb_config(portid_t pid,
3123                      enum dcb_mode_enable dcb_mode,
3124                      enum rte_eth_nb_tcs num_tcs,
3125                      uint8_t pfc_en)
3126 {
3127         struct rte_eth_conf port_conf;
3128         struct rte_port *rte_port;
3129         int retval;
3130         uint16_t i;
3131
3132         rte_port = &ports[pid];
3133
3134         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3135         /* Enter DCB configuration status */
3136         dcb_config = 1;
3137
3138         port_conf.rxmode = rte_port->dev_conf.rxmode;
3139         port_conf.txmode = rte_port->dev_conf.txmode;
3140
3141         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3142         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3143         if (retval < 0)
3144                 return retval;
3145         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3146
3147         /* re-configure the device . */
3148         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3149         if (retval < 0)
3150                 return retval;
3151
3152         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3153         if (retval != 0)
3154                 return retval;
3155
3156         /* If dev_info.vmdq_pool_base is greater than 0,
3157          * the queue id of vmdq pools is started after pf queues.
3158          */
3159         if (dcb_mode == DCB_VT_ENABLED &&
3160             rte_port->dev_info.vmdq_pool_base > 0) {
3161                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3162                         " for port %d.", pid);
3163                 return -1;
3164         }
3165
3166         /* Assume the ports in testpmd have the same dcb capability
3167          * and has the same number of rxq and txq in dcb mode
3168          */
3169         if (dcb_mode == DCB_VT_ENABLED) {
3170                 if (rte_port->dev_info.max_vfs > 0) {
3171                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3172                         nb_txq = rte_port->dev_info.nb_tx_queues;
3173                 } else {
3174                         nb_rxq = rte_port->dev_info.max_rx_queues;
3175                         nb_txq = rte_port->dev_info.max_tx_queues;
3176                 }
3177         } else {
3178                 /*if vt is disabled, use all pf queues */
3179                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3180                         nb_rxq = rte_port->dev_info.max_rx_queues;
3181                         nb_txq = rte_port->dev_info.max_tx_queues;
3182                 } else {
3183                         nb_rxq = (queueid_t)num_tcs;
3184                         nb_txq = (queueid_t)num_tcs;
3185
3186                 }
3187         }
3188         rx_free_thresh = 64;
3189
3190         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3191
3192         rxtx_port_config(rte_port);
3193         /* VLAN filter */
3194         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3195         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3196                 rx_vft_set(pid, vlan_tags[i], 1);
3197
3198         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3199         if (retval != 0)
3200                 return retval;
3201
3202         map_port_queue_stats_mapping_registers(pid, rte_port);
3203
3204         rte_port->dcb_flag = 1;
3205
3206         return 0;
3207 }
3208
3209 static void
3210 init_port(void)
3211 {
3212         /* Configuration of Ethernet ports. */
3213         ports = rte_zmalloc("testpmd: ports",
3214                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3215                             RTE_CACHE_LINE_SIZE);
3216         if (ports == NULL) {
3217                 rte_exit(EXIT_FAILURE,
3218                                 "rte_zmalloc(%d struct rte_port) failed\n",
3219                                 RTE_MAX_ETHPORTS);
3220         }
3221
3222         /* Initialize ports NUMA structures */
3223         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3224         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3225         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3226 }
3227
3228 static void
3229 force_quit(void)
3230 {
3231         pmd_test_exit();
3232         prompt_exit();
3233 }
3234
3235 static void
3236 print_stats(void)
3237 {
3238         uint8_t i;
3239         const char clr[] = { 27, '[', '2', 'J', '\0' };
3240         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3241
3242         /* Clear screen and move to top left */
3243         printf("%s%s", clr, top_left);
3244
3245         printf("\nPort statistics ====================================");
3246         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3247                 nic_stats_display(fwd_ports_ids[i]);
3248
3249         fflush(stdout);
3250 }
3251
3252 static void
3253 signal_handler(int signum)
3254 {
3255         if (signum == SIGINT || signum == SIGTERM) {
3256                 printf("\nSignal %d received, preparing to exit...\n",
3257                                 signum);
3258 #ifdef RTE_LIBRTE_PDUMP
3259                 /* uninitialize packet capture framework */
3260                 rte_pdump_uninit();
3261 #endif
3262 #ifdef RTE_LIBRTE_LATENCY_STATS
3263                 if (latencystats_enabled != 0)
3264                         rte_latencystats_uninit();
3265 #endif
3266                 force_quit();
3267                 /* Set flag to indicate the force termination. */
3268                 f_quit = 1;
3269                 /* exit with the expected status */
3270                 signal(signum, SIG_DFL);
3271                 kill(getpid(), signum);
3272         }
3273 }
3274
3275 int
3276 main(int argc, char** argv)
3277 {
3278         int diag;
3279         portid_t port_id;
3280         uint16_t count;
3281         int ret;
3282
3283         signal(SIGINT, signal_handler);
3284         signal(SIGTERM, signal_handler);
3285
3286         testpmd_logtype = rte_log_register("testpmd");
3287         if (testpmd_logtype < 0)
3288                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3289         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3290
3291         diag = rte_eal_init(argc, argv);
3292         if (diag < 0)
3293                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3294                          rte_strerror(rte_errno));
3295
3296         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3297                 rte_exit(EXIT_FAILURE,
3298                          "Secondary process type not supported.\n");
3299
3300         ret = register_eth_event_callback();
3301         if (ret != 0)
3302                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3303
3304 #ifdef RTE_LIBRTE_PDUMP
3305         /* initialize packet capture framework */
3306         rte_pdump_init();
3307 #endif
3308
3309         count = 0;
3310         RTE_ETH_FOREACH_DEV(port_id) {
3311                 ports_ids[count] = port_id;
3312                 count++;
3313         }
3314         nb_ports = (portid_t) count;
3315         if (nb_ports == 0)
3316                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3317
3318         /* allocate port structures, and init them */
3319         init_port();
3320
3321         set_def_fwd_config();
3322         if (nb_lcores == 0)
3323                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3324                          "Check the core mask argument\n");
3325
3326         /* Bitrate/latency stats disabled by default */
3327 #ifdef RTE_LIBRTE_BITRATE
3328         bitrate_enabled = 0;
3329 #endif
3330 #ifdef RTE_LIBRTE_LATENCY_STATS
3331         latencystats_enabled = 0;
3332 #endif
3333
3334         /* on FreeBSD, mlockall() is disabled by default */
3335 #ifdef RTE_EXEC_ENV_FREEBSD
3336         do_mlockall = 0;
3337 #else
3338         do_mlockall = 1;
3339 #endif
3340
3341         argc -= diag;
3342         argv += diag;
3343         if (argc > 1)
3344                 launch_args_parse(argc, argv);
3345
3346         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3347                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3348                         strerror(errno));
3349         }
3350
3351         if (tx_first && interactive)
3352                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3353                                 "interactive mode.\n");
3354
3355         if (tx_first && lsc_interrupt) {
3356                 printf("Warning: lsc_interrupt needs to be off when "
3357                                 " using tx_first. Disabling.\n");
3358                 lsc_interrupt = 0;
3359         }
3360
3361         if (!nb_rxq && !nb_txq)
3362                 printf("Warning: Either rx or tx queues should be non-zero\n");
3363
3364         if (nb_rxq > 1 && nb_rxq > nb_txq)
3365                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3366                        "but nb_txq=%d will prevent to fully test it.\n",
3367                        nb_rxq, nb_txq);
3368
3369         init_config();
3370
3371         if (hot_plug) {
3372                 ret = rte_dev_hotplug_handle_enable();
3373                 if (ret) {
3374                         RTE_LOG(ERR, EAL,
3375                                 "fail to enable hotplug handling.");
3376                         return -1;
3377                 }
3378
3379                 ret = rte_dev_event_monitor_start();
3380                 if (ret) {
3381                         RTE_LOG(ERR, EAL,
3382                                 "fail to start device event monitoring.");
3383                         return -1;
3384                 }
3385
3386                 ret = rte_dev_event_callback_register(NULL,
3387                         dev_event_callback, NULL);
3388                 if (ret) {
3389                         RTE_LOG(ERR, EAL,
3390                                 "fail  to register device event callback\n");
3391                         return -1;
3392                 }
3393         }
3394
3395         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3396                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3397
3398         /* set all ports to promiscuous mode by default */
3399         RTE_ETH_FOREACH_DEV(port_id) {
3400                 ret = rte_eth_promiscuous_enable(port_id);
3401                 if (ret != 0)
3402                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3403                                 port_id, rte_strerror(-ret));
3404         }
3405
3406         /* Init metrics library */
3407         rte_metrics_init(rte_socket_id());
3408
3409 #ifdef RTE_LIBRTE_LATENCY_STATS
3410         if (latencystats_enabled != 0) {
3411                 int ret = rte_latencystats_init(1, NULL);
3412                 if (ret)
3413                         printf("Warning: latencystats init()"
3414                                 " returned error %d\n", ret);
3415                 printf("Latencystats running on lcore %d\n",
3416                         latencystats_lcore_id);
3417         }
3418 #endif
3419
3420         /* Setup bitrate stats */
3421 #ifdef RTE_LIBRTE_BITRATE
3422         if (bitrate_enabled != 0) {
3423                 bitrate_data = rte_stats_bitrate_create();
3424                 if (bitrate_data == NULL)
3425                         rte_exit(EXIT_FAILURE,
3426                                 "Could not allocate bitrate data.\n");
3427                 rte_stats_bitrate_reg(bitrate_data);
3428         }
3429 #endif
3430
3431 #ifdef RTE_LIBRTE_CMDLINE
3432         if (strlen(cmdline_filename) != 0)
3433                 cmdline_read_from_file(cmdline_filename);
3434
3435         if (interactive == 1) {
3436                 if (auto_start) {
3437                         printf("Start automatic packet forwarding\n");
3438                         start_packet_forwarding(0);
3439                 }
3440                 prompt();
3441                 pmd_test_exit();
3442         } else
3443 #endif
3444         {
3445                 char c;
3446                 int rc;
3447
3448                 f_quit = 0;
3449
3450                 printf("No commandline core given, start packet forwarding\n");
3451                 start_packet_forwarding(tx_first);
3452                 if (stats_period != 0) {
3453                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3454                         uint64_t timer_period;
3455
3456                         /* Convert to number of cycles */
3457                         timer_period = stats_period * rte_get_timer_hz();
3458
3459                         while (f_quit == 0) {
3460                                 cur_time = rte_get_timer_cycles();
3461                                 diff_time += cur_time - prev_time;
3462
3463                                 if (diff_time >= timer_period) {
3464                                         print_stats();
3465                                         /* Reset the timer */
3466                                         diff_time = 0;
3467                                 }
3468                                 /* Sleep to avoid unnecessary checks */
3469                                 prev_time = cur_time;
3470                                 sleep(1);
3471                         }
3472                 }
3473
3474                 printf("Press enter to exit\n");
3475                 rc = read(0, &c, 1);
3476                 pmd_test_exit();
3477                 if (rc < 0)
3478                         return 1;
3479         }
3480
3481         return 0;
3482 }