app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_malloc_heap.h>
  31 #include <rte_memory.h>
  32 #include <rte_memcpy.h>
  33 #include <rte_launch.h>
  34 #include <rte_eal.h>
  35 #include <rte_alarm.h>
  36 #include <rte_per_lcore.h>
  37 #include <rte_lcore.h>
  38 #include <rte_atomic.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_LIBRTE_IXGBE_PMD
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIBRTE_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #include <rte_metrics.h>
  58 #ifdef RTE_LIBRTE_BITRATE
  59 #include <rte_bitrate.h>
  60 #endif
  61 #ifdef RTE_LIBRTE_LATENCY_STATS
  62 #include <rte_latencystats.h>
  63 #endif
  64
  65 #include "testpmd.h"
  66
  67 #ifndef MAP_HUGETLB
  68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  69 #define HUGE_FLAG (0x40000)
  70 #else
  71 #define HUGE_FLAG MAP_HUGETLB
  72 #endif
  73
  74 #ifndef MAP_HUGE_SHIFT
  75 /* older kernels (or FreeBSD) will not have this define */
  76 #define HUGE_SHIFT (26)
  77 #else
  78 #define HUGE_SHIFT MAP_HUGE_SHIFT
  79 #endif
  80
  81 #define EXTMEM_HEAP_NAME "extmem"
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192 uint16_t mempool_flags;
 193
 194 struct fwd_config cur_fwd_config;
 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 196 uint32_t retry_enabled;
 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 199
 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 201 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 202                                       * specified on command-line. */
 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 204
 205 /*
 206  * In container, it cannot terminate the process which running with 'stats-period'
 207  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 208  */
 209 uint8_t f_quit;
 210
 211 /*
 212  * Configuration of packet segments used by the "txonly" processing engine.
 213  */
 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 216         TXONLY_DEF_PACKET_LEN,
 217 };
 218 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 219
 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 221 /**< Split policy for packets to TX. */
 222
 223 uint8_t txonly_multi_flow;
 224 /**< Whether multiple flows are generated in TXONLY mode. */
 225
 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 228
 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 230 uint8_t dcb_config = 0;
 231
 232 /* Whether the dcb is in testing status */
 233 uint8_t dcb_test = 0;
 234
 235 /*
 236  * Configurable number of RX/TX queues.
 237  */
 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 240
 241 /*
 242  * Configurable number of RX/TX ring descriptors.
 243  * Defaults are supplied by drivers via ethdev.
 244  */
 245 #define RTE_TEST_RX_DESC_DEFAULT 0
 246 #define RTE_TEST_TX_DESC_DEFAULT 0
 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 249
 250 #define RTE_PMD_PARAM_UNSET -1
 251 /*
 252  * Configurable values of RX and TX ring threshold registers.
 253  */
 254
 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 258
 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 262
 263 /*
 264  * Configurable value of RX free threshold.
 265  */
 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 267
 268 /*
 269  * Configurable value of RX drop enable.
 270  */
 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 272
 273 /*
 274  * Configurable value of TX free threshold.
 275  */
 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of TX RS bit threshold.
 280  */
 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of buffered packets before sending.
 285  */
 286 uint16_t noisy_tx_sw_bufsz;
 287
 288 /*
 289  * Configurable value of packet buffer timeout.
 290  */
 291 uint16_t noisy_tx_sw_buf_flush_time;
 292
 293 /*
 294  * Configurable value for size of VNF internal memory area
 295  * used for simulating noisy neighbour behaviour
 296  */
 297 uint64_t noisy_lkup_mem_sz;
 298
 299 /*
 300  * Configurable value of number of random writes done in
 301  * VNF simulation memory area.
 302  */
 303 uint64_t noisy_lkup_num_writes;
 304
 305 /*
 306  * Configurable value of number of random reads done in
 307  * VNF simulation memory area.
 308  */
 309 uint64_t noisy_lkup_num_reads;
 310
 311 /*
 312  * Configurable value of number of random reads/writes done in
 313  * VNF simulation memory area.
 314  */
 315 uint64_t noisy_lkup_num_reads_writes;
 316
 317 /*
 318  * Receive Side Scaling (RSS) configuration.
 319  */
 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 321
 322 /*
 323  * Port topology configuration
 324  */
 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 326
 327 /*
 328  * Avoids to flush all the RX streams before starts forwarding.
 329  */
 330 uint8_t no_flush_rx = 0; /* flush by default */
 331
 332 /*
 333  * Flow API isolated mode.
 334  */
 335 uint8_t flow_isolate_all;
 336
 337 /*
 338  * Avoids to check link status when starting/stopping a port.
 339  */
 340 uint8_t no_link_check = 0; /* check by default */
 341
 342 /*
 343  * Don't automatically start all ports in interactive mode.
 344  */
 345 uint8_t no_device_start = 0;
 346
 347 /*
 348  * Enable link status change notification
 349  */
 350 uint8_t lsc_interrupt = 1; /* enabled by default */
 351
 352 /*
 353  * Enable device removal notification.
 354  */
 355 uint8_t rmv_interrupt = 1; /* enabled by default */
 356
 357 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 358
 359 /* After attach, port setup is called on event or by iterator */
 360 bool setup_on_probe_event = true;
 361
 362 /* Pretty printing of ethdev events */
 363 static const char * const eth_event_desc[] = {
 364         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 365         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 366         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 367         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 368         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 369         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 370         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 371         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 372         [RTE_ETH_EVENT_NEW] = "device probed",
 373         [RTE_ETH_EVENT_DESTROY] = "device released",
 374         [RTE_ETH_EVENT_MAX] = NULL,
 375 };
 376
 377 /*
 378  * Display or mask ether events
 379  * Default to all events except VF_MBOX
 380  */
 381 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 382                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 383                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 384                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 385                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 386                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 387                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 388 /*
 389  * Decide if all memory are locked for performance.
 390  */
 391 int do_mlockall = 0;
 392
 393 /*
 394  * NIC bypass mode configuration options.
 395  */
 396
 397 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 398 /* The NIC bypass watchdog timeout. */
 399 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 400 #endif
 401
 402
 403 #ifdef RTE_LIBRTE_LATENCY_STATS
 404
 405 /*
 406  * Set when latency stats is enabled in the commandline
 407  */
 408 uint8_t latencystats_enabled;
 409
 410 /*
 411  * Lcore ID to serive latency statistics.
 412  */
 413 lcoreid_t latencystats_lcore_id = -1;
 414
 415 #endif
 416
 417 /*
 418  * Ethernet device configuration.
 419  */
 420 struct rte_eth_rxmode rx_mode = {
 421         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 422                 /**< Default maximum frame length. */
 423 };
 424
 425 struct rte_eth_txmode tx_mode = {
 426         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 427 };
 428
 429 struct rte_fdir_conf fdir_conf = {
 430         .mode = RTE_FDIR_MODE_NONE,
 431         .pballoc = RTE_FDIR_PBALLOC_64K,
 432         .status = RTE_FDIR_REPORT_STATUS,
 433         .mask = {
 434                 .vlan_tci_mask = 0xFFEF,
 435                 .ipv4_mask     = {
 436                         .src_ip = 0xFFFFFFFF,
 437                         .dst_ip = 0xFFFFFFFF,
 438                 },
 439                 .ipv6_mask     = {
 440                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 441                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 442                 },
 443                 .src_port_mask = 0xFFFF,
 444                 .dst_port_mask = 0xFFFF,
 445                 .mac_addr_byte_mask = 0xFF,
 446                 .tunnel_type_mask = 1,
 447                 .tunnel_id_mask = 0xFFFFFFFF,
 448         },
 449         .drop_queue = 127,
 450 };
 451
 452 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 453
 454 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 455 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 456
 457 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 458 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 459
 460 uint16_t nb_tx_queue_stats_mappings = 0;
 461 uint16_t nb_rx_queue_stats_mappings = 0;
 462
 463 /*
 464  * Display zero values by default for xstats
 465  */
 466 uint8_t xstats_hide_zero;
 467
 468 unsigned int num_sockets = 0;
 469 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 470
 471 #ifdef RTE_LIBRTE_BITRATE
 472 /* Bitrate statistics */
 473 struct rte_stats_bitrates *bitrate_data;
 474 lcoreid_t bitrate_lcore_id;
 475 uint8_t bitrate_enabled;
 476 #endif
 477
 478 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 479 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 480
 481 struct vxlan_encap_conf vxlan_encap_conf = {
 482         .select_ipv4 = 1,
 483         .select_vlan = 0,
 484         .select_tos_ttl = 0,
 485         .vni = "\x00\x00\x00",
 486         .udp_src = 0,
 487         .udp_dst = RTE_BE16(4789),
 488         .ipv4_src = RTE_IPV4(127, 0, 0, 1),
 489         .ipv4_dst = RTE_IPV4(255, 255, 255, 255),
 490         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 491                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 492         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 493                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 494         .vlan_tci = 0,
 495         .ip_tos = 0,
 496         .ip_ttl = 255,
 497         .eth_src = "\x00\x00\x00\x00\x00\x00",
 498         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 499 };
 500
 501 struct nvgre_encap_conf nvgre_encap_conf = {
 502         .select_ipv4 = 1,
 503         .select_vlan = 0,
 504         .tni = "\x00\x00\x00",
 505         .ipv4_src = RTE_IPV4(127, 0, 0, 1),
 506         .ipv4_dst = RTE_IPV4(255, 255, 255, 255),
 507         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 508                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 509         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 510                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 511         .vlan_tci = 0,
 512         .eth_src = "\x00\x00\x00\x00\x00\x00",
 513         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 514 };
 515
 516 /* Forward function declarations */
 517 static void setup_attached_port(portid_t pi);
 518 static void map_port_queue_stats_mapping_registers(portid_t pi,
 519                                                    struct rte_port *port);
 520 static void check_all_ports_link_status(uint32_t port_mask);
 521 static int eth_event_callback(portid_t port_id,
 522                               enum rte_eth_event_type type,
 523                               void *param, void *ret_param);
 524 static void dev_event_callback(const char *device_name,
 525                                 enum rte_dev_event_type type,
 526                                 void *param);
 527
 528 /*
 529  * Check if all the ports are started.
 530  * If yes, return positive value. If not, return zero.
 531  */
 532 static int all_ports_started(void);
 533
 534 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 535 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 536
 537 /*
 538  * Helper function to check if socket is already discovered.
 539  * If yes, return positive value. If not, return zero.
 540  */
 541 int
 542 new_socket_id(unsigned int socket_id)
 543 {
 544         unsigned int i;
 545
 546         for (i = 0; i < num_sockets; i++) {
 547                 if (socket_ids[i] == socket_id)
 548                         return 0;
 549         }
 550         return 1;
 551 }
 552
 553 /*
 554  * Setup default configuration.
 555  */
 556 static void
 557 set_default_fwd_lcores_config(void)
 558 {
 559         unsigned int i;
 560         unsigned int nb_lc;
 561         unsigned int sock_num;
 562
 563         nb_lc = 0;
 564         for (i = 0; i < RTE_MAX_LCORE; i++) {
 565                 if (!rte_lcore_is_enabled(i))
 566                         continue;
 567                 sock_num = rte_lcore_to_socket_id(i);
 568                 if (new_socket_id(sock_num)) {
 569                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 570                                 rte_exit(EXIT_FAILURE,
 571                                          "Total sockets greater than %u\n",
 572                                          RTE_MAX_NUMA_NODES);
 573                         }
 574                         socket_ids[num_sockets++] = sock_num;
 575                 }
 576                 if (i == rte_get_master_lcore())
 577                         continue;
 578                 fwd_lcores_cpuids[nb_lc++] = i;
 579         }
 580         nb_lcores = (lcoreid_t) nb_lc;
 581         nb_cfg_lcores = nb_lcores;
 582         nb_fwd_lcores = 1;
 583 }
 584
 585 static void
 586 set_def_peer_eth_addrs(void)
 587 {
 588         portid_t i;
 589
 590         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 591                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 592                 peer_eth_addrs[i].addr_bytes[5] = i;
 593         }
 594 }
 595
 596 static void
 597 set_default_fwd_ports_config(void)
 598 {
 599         portid_t pt_id;
 600         int i = 0;
 601
 602         RTE_ETH_FOREACH_DEV(pt_id) {
 603                 fwd_ports_ids[i++] = pt_id;
 604
 605                 /* Update sockets info according to the attached device */
 606                 int socket_id = rte_eth_dev_socket_id(pt_id);
 607                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 608                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 609                                 rte_exit(EXIT_FAILURE,
 610                                          "Total sockets greater than %u\n",
 611                                          RTE_MAX_NUMA_NODES);
 612                         }
 613                         socket_ids[num_sockets++] = socket_id;
 614                 }
 615         }
 616
 617         nb_cfg_ports = nb_ports;
 618         nb_fwd_ports = nb_ports;
 619 }
 620
 621 void
 622 set_def_fwd_config(void)
 623 {
 624         set_default_fwd_lcores_config();
 625         set_def_peer_eth_addrs();
 626         set_default_fwd_ports_config();
 627 }
 628
 629 /* extremely pessimistic estimation of memory required to create a mempool */
 630 static int
 631 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 632 {
 633         unsigned int n_pages, mbuf_per_pg, leftover;
 634         uint64_t total_mem, mbuf_mem, obj_sz;
 635
 636         /* there is no good way to predict how much space the mempool will
 637          * occupy because it will allocate chunks on the fly, and some of those
 638          * will come from default DPDK memory while some will come from our
 639          * external memory, so just assume 128MB will be enough for everyone.
 640          */
 641         uint64_t hdr_mem = 128 << 20;
 642
 643         /* account for possible non-contiguousness */
 644         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 645         if (obj_sz > pgsz) {
 646                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 647                 return -1;
 648         }
 649
 650         mbuf_per_pg = pgsz / obj_sz;
 651         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 652         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 653
 654         mbuf_mem = n_pages * pgsz;
 655
 656         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 657
 658         if (total_mem > SIZE_MAX) {
 659                 TESTPMD_LOG(ERR, "Memory size too big\n");
 660                 return -1;
 661         }
 662         *out = (size_t)total_mem;
 663
 664         return 0;
 665 }
 666
 667 static int
 668 pagesz_flags(uint64_t page_sz)
 669 {
 670         /* as per mmap() manpage, all page sizes are log2 of page size
 671          * shifted by MAP_HUGE_SHIFT
 672          */
 673         int log2 = rte_log2_u64(page_sz);
 674
 675         return (log2 << HUGE_SHIFT);
 676 }
 677
 678 static void *
 679 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 680 {
 681         void *addr;
 682         int flags;
 683
 684         /* allocate anonymous hugepages */
 685         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 686         if (huge)
 687                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 688
 689         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 690         if (addr == MAP_FAILED)
 691                 return NULL;
 692
 693         return addr;
 694 }
 695
 696 struct extmem_param {
 697         void *addr;
 698         size_t len;
 699         size_t pgsz;
 700         rte_iova_t *iova_table;
 701         unsigned int iova_table_len;
 702 };
 703
 704 static int
 705 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 706                 bool huge)
 707 {
 708         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 709                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 710         unsigned int cur_page, n_pages, pgsz_idx;
 711         size_t mem_sz, cur_pgsz;
 712         rte_iova_t *iovas = NULL;
 713         void *addr;
 714         int ret;
 715
 716         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 717                 /* skip anything that is too big */
 718                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 719                         continue;
 720
 721                 cur_pgsz = pgsizes[pgsz_idx];
 722
 723                 /* if we were told not to allocate hugepages, override */
 724                 if (!huge)
 725                         cur_pgsz = sysconf(_SC_PAGESIZE);
 726
 727                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 728                 if (ret < 0) {
 729                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 730                         return -1;
 731                 }
 732
 733                 /* allocate our memory */
 734                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 735
 736                 /* if we couldn't allocate memory with a specified page size,
 737                  * that doesn't mean we can't do it with other page sizes, so
 738                  * try another one.
 739                  */
 740                 if (addr == NULL)
 741                         continue;
 742
 743                 /* store IOVA addresses for every page in this memory area */
 744                 n_pages = mem_sz / cur_pgsz;
 745
 746                 iovas = malloc(sizeof(*iovas) * n_pages);
 747
 748                 if (iovas == NULL) {
 749                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 750                         goto fail;
 751                 }
 752                 /* lock memory if it's not huge pages */
 753                 if (!huge)
 754                         mlock(addr, mem_sz);
 755
 756                 /* populate IOVA addresses */
 757                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 758                         rte_iova_t iova;
 759                         size_t offset;
 760                         void *cur;
 761
 762                         offset = cur_pgsz * cur_page;
 763                         cur = RTE_PTR_ADD(addr, offset);
 764
 765                         /* touch the page before getting its IOVA */
 766                         *(volatile char *)cur = 0;
 767
 768                         iova = rte_mem_virt2iova(cur);
 769
 770                         iovas[cur_page] = iova;
 771                 }
 772
 773                 break;
 774         }
 775         /* if we couldn't allocate anything */
 776         if (iovas == NULL)
 777                 return -1;
 778
 779         param->addr = addr;
 780         param->len = mem_sz;
 781         param->pgsz = cur_pgsz;
 782         param->iova_table = iovas;
 783         param->iova_table_len = n_pages;
 784
 785         return 0;
 786 fail:
 787         if (iovas)
 788                 free(iovas);
 789         if (addr)
 790                 munmap(addr, mem_sz);
 791
 792         return -1;
 793 }
 794
 795 static int
 796 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 797 {
 798         struct extmem_param param;
 799         int socket_id, ret;
 800
 801         memset(&param, 0, sizeof(param));
 802
 803         /* check if our heap exists */
 804         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 805         if (socket_id < 0) {
 806                 /* create our heap */
 807                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 808                 if (ret < 0) {
 809                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 810                         return -1;
 811                 }
 812         }
 813
 814         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 815         if (ret < 0) {
 816                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 817                 return -1;
 818         }
 819
 820         /* we now have a valid memory area, so add it to heap */
 821         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 822                         param.addr, param.len, param.iova_table,
 823                         param.iova_table_len, param.pgsz);
 824
 825         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 826
 827         /* not needed any more */
 828         free(param.iova_table);
 829
 830         if (ret < 0) {
 831                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 832                 munmap(param.addr, param.len);
 833                 return -1;
 834         }
 835
 836         /* success */
 837
 838         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 839                         param.len >> 20);
 840
 841         return 0;
 842 }
 843 static void
 844 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 845              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 846 {
 847         uint16_t pid = 0;
 848         int ret;
 849
 850         RTE_ETH_FOREACH_DEV(pid) {
 851                 struct rte_eth_dev *dev =
 852                         &rte_eth_devices[pid];
 853
 854                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 855                                         memhdr->len);
 856                 if (ret) {
 857                         TESTPMD_LOG(DEBUG,
 858                                     "unable to DMA unmap addr 0x%p "
 859                                     "for device %s\n",
 860                                     memhdr->addr, dev->data->name);
 861                 }
 862         }
 863         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 864         if (ret) {
 865                 TESTPMD_LOG(DEBUG,
 866                             "unable to un-register addr 0x%p\n", memhdr->addr);
 867         }
 868 }
 869
 870 static void
 871 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 872            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 873 {
 874         uint16_t pid = 0;
 875         size_t page_size = sysconf(_SC_PAGESIZE);
 876         int ret;
 877
 878         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 879                                   page_size);
 880         if (ret) {
 881                 TESTPMD_LOG(DEBUG,
 882                             "unable to register addr 0x%p\n", memhdr->addr);
 883                 return;
 884         }
 885         RTE_ETH_FOREACH_DEV(pid) {
 886                 struct rte_eth_dev *dev =
 887                         &rte_eth_devices[pid];
 888
 889                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 890                                       memhdr->len);
 891                 if (ret) {
 892                         TESTPMD_LOG(DEBUG,
 893                                     "unable to DMA map addr 0x%p "
 894                                     "for device %s\n",
 895                                     memhdr->addr, dev->data->name);
 896                 }
 897         }
 898 }
 899
 900 /*
 901  * Configuration initialisation done once at init time.
 902  */
 903 static struct rte_mempool *
 904 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 905                  unsigned int socket_id)
 906 {
 907         char pool_name[RTE_MEMPOOL_NAMESIZE];
 908         struct rte_mempool *rte_mp = NULL;
 909         uint32_t mb_size;
 910
 911         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 912         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 913
 914         TESTPMD_LOG(INFO,
 915                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 916                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 917
 918         switch (mp_alloc_type) {
 919         case MP_ALLOC_NATIVE:
 920                 {
 921                         /* wrapper to rte_mempool_create() */
 922                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 923                                         rte_mbuf_best_mempool_ops());
 924                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 925                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 926                         break;
 927                 }
 928         case MP_ALLOC_ANON:
 929                 {
 930                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 931                                 mb_size, (unsigned int) mb_mempool_cache,
 932                                 sizeof(struct rte_pktmbuf_pool_private),
 933                                 socket_id, mempool_flags);
 934                         if (rte_mp == NULL)
 935                                 goto err;
 936
 937                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 938                                 rte_mempool_free(rte_mp);
 939                                 rte_mp = NULL;
 940                                 goto err;
 941                         }
 942                         rte_pktmbuf_pool_init(rte_mp, NULL);
 943                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 944                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 945                         break;
 946                 }
 947         case MP_ALLOC_XMEM:
 948         case MP_ALLOC_XMEM_HUGE:
 949                 {
 950                         int heap_socket;
 951                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 952
 953                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 954                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 955
 956                         heap_socket =
 957                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 958                         if (heap_socket < 0)
 959                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 960
 961                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 962                                         rte_mbuf_best_mempool_ops());
 963                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 964                                         mb_mempool_cache, 0, mbuf_seg_size,
 965                                         heap_socket);
 966                         break;
 967                 }
 968         default:
 969                 {
 970                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 971                 }
 972         }
 973
 974 err:
 975         if (rte_mp == NULL) {
 976                 rte_exit(EXIT_FAILURE,
 977                         "Creation of mbuf pool for socket %u failed: %s\n",
 978                         socket_id, rte_strerror(rte_errno));
 979         } else if (verbose_level > 0) {
 980                 rte_mempool_dump(stdout, rte_mp);
 981         }
 982         return rte_mp;
 983 }
 984
 985 /*
 986  * Check given socket id is valid or not with NUMA mode,
 987  * if valid, return 0, else return -1
 988  */
 989 static int
 990 check_socket_id(const unsigned int socket_id)
 991 {
 992         static int warning_once = 0;
 993
 994         if (new_socket_id(socket_id)) {
 995                 if (!warning_once && numa_support)
 996                         printf("Warning: NUMA should be configured manually by"
 997                                " using --port-numa-config and"
 998                                " --ring-numa-config parameters along with"
 999                                " --numa.\n");
1000                 warning_once = 1;
1001                 return -1;
1002         }
1003         return 0;
1004 }
1005
1006 /*
1007  * Get the allowed maximum number of RX queues.
1008  * *pid return the port id which has minimal value of
1009  * max_rx_queues in all ports.
1010  */
1011 queueid_t
1012 get_allowed_max_nb_rxq(portid_t *pid)
1013 {
1014         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
1015         portid_t pi;
1016         struct rte_eth_dev_info dev_info;
1017
1018         RTE_ETH_FOREACH_DEV(pi) {
1019                 rte_eth_dev_info_get(pi, &dev_info);
1020                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1021                         allowed_max_rxq = dev_info.max_rx_queues;
1022                         *pid = pi;
1023                 }
1024         }
1025         return allowed_max_rxq;
1026 }
1027
1028 /*
1029  * Check input rxq is valid or not.
1030  * If input rxq is not greater than any of maximum number
1031  * of RX queues of all ports, it is valid.
1032  * if valid, return 0, else return -1
1033  */
1034 int
1035 check_nb_rxq(queueid_t rxq)
1036 {
1037         queueid_t allowed_max_rxq;
1038         portid_t pid = 0;
1039
1040         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1041         if (rxq > allowed_max_rxq) {
1042                 printf("Fail: input rxq (%u) can't be greater "
1043                        "than max_rx_queues (%u) of port %u\n",
1044                        rxq,
1045                        allowed_max_rxq,
1046                        pid);
1047                 return -1;
1048         }
1049         return 0;
1050 }
1051
1052 /*
1053  * Get the allowed maximum number of TX queues.
1054  * *pid return the port id which has minimal value of
1055  * max_tx_queues in all ports.
1056  */
1057 queueid_t
1058 get_allowed_max_nb_txq(portid_t *pid)
1059 {
1060         queueid_t allowed_max_txq = MAX_QUEUE_ID;
1061         portid_t pi;
1062         struct rte_eth_dev_info dev_info;
1063
1064         RTE_ETH_FOREACH_DEV(pi) {
1065                 rte_eth_dev_info_get(pi, &dev_info);
1066                 if (dev_info.max_tx_queues < allowed_max_txq) {
1067                         allowed_max_txq = dev_info.max_tx_queues;
1068                         *pid = pi;
1069                 }
1070         }
1071         return allowed_max_txq;
1072 }
1073
1074 /*
1075  * Check input txq is valid or not.
1076  * If input txq is not greater than any of maximum number
1077  * of TX queues of all ports, it is valid.
1078  * if valid, return 0, else return -1
1079  */
1080 int
1081 check_nb_txq(queueid_t txq)
1082 {
1083         queueid_t allowed_max_txq;
1084         portid_t pid = 0;
1085
1086         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1087         if (txq > allowed_max_txq) {
1088                 printf("Fail: input txq (%u) can't be greater "
1089                        "than max_tx_queues (%u) of port %u\n",
1090                        txq,
1091                        allowed_max_txq,
1092                        pid);
1093                 return -1;
1094         }
1095         return 0;
1096 }
1097
1098 static void
1099 init_config(void)
1100 {
1101         portid_t pid;
1102         struct rte_port *port;
1103         struct rte_mempool *mbp;
1104         unsigned int nb_mbuf_per_pool;
1105         lcoreid_t  lc_id;
1106         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1107         struct rte_gro_param gro_param;
1108         uint32_t gso_types;
1109         uint16_t data_size;
1110         bool warning = 0;
1111         int k;
1112
1113         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1114
1115         /* Configuration of logical cores. */
1116         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1117                                 sizeof(struct fwd_lcore *) * nb_lcores,
1118                                 RTE_CACHE_LINE_SIZE);
1119         if (fwd_lcores == NULL) {
1120                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1121                                                         "failed\n", nb_lcores);
1122         }
1123         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1124                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1125                                                sizeof(struct fwd_lcore),
1126                                                RTE_CACHE_LINE_SIZE);
1127                 if (fwd_lcores[lc_id] == NULL) {
1128                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1129                                                                 "failed\n");
1130                 }
1131                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1132         }
1133
1134         RTE_ETH_FOREACH_DEV(pid) {
1135                 port = &ports[pid];
1136                 /* Apply default TxRx configuration for all ports */
1137                 port->dev_conf.txmode = tx_mode;
1138                 port->dev_conf.rxmode = rx_mode;
1139                 rte_eth_dev_info_get(pid, &port->dev_info);
1140
1141                 if (!(port->dev_info.tx_offload_capa &
1142                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1143                         port->dev_conf.txmode.offloads &=
1144                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1145                 if (!(port->dev_info.tx_offload_capa &
1146                         DEV_TX_OFFLOAD_MATCH_METADATA))
1147                         port->dev_conf.txmode.offloads &=
1148                                 ~DEV_TX_OFFLOAD_MATCH_METADATA;
1149                 if (numa_support) {
1150                         if (port_numa[pid] != NUMA_NO_CONFIG)
1151                                 port_per_socket[port_numa[pid]]++;
1152                         else {
1153                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1154
1155                                 /*
1156                                  * if socket_id is invalid,
1157                                  * set to the first available socket.
1158                                  */
1159                                 if (check_socket_id(socket_id) < 0)
1160                                         socket_id = socket_ids[0];
1161                                 port_per_socket[socket_id]++;
1162                         }
1163                 }
1164
1165                 /* Apply Rx offloads configuration */
1166                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1167                         port->rx_conf[k].offloads =
1168                                 port->dev_conf.rxmode.offloads;
1169                 /* Apply Tx offloads configuration */
1170                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1171                         port->tx_conf[k].offloads =
1172                                 port->dev_conf.txmode.offloads;
1173
1174                 /* set flag to initialize port/queue */
1175                 port->need_reconfig = 1;
1176                 port->need_reconfig_queues = 1;
1177                 port->tx_metadata = 0;
1178
1179                 /* Check for maximum number of segments per MTU. Accordingly
1180                  * update the mbuf data size.
1181                  */
1182                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1183                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1184                         data_size = rx_mode.max_rx_pkt_len /
1185                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1186
1187                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1188                                                         mbuf_data_size) {
1189                                 mbuf_data_size = data_size +
1190                                                  RTE_PKTMBUF_HEADROOM;
1191                                 warning = 1;
1192                         }
1193                 }
1194         }
1195
1196         if (warning)
1197                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1198                             mbuf_data_size);
1199
1200         /*
1201          * Create pools of mbuf.
1202          * If NUMA support is disabled, create a single pool of mbuf in
1203          * socket 0 memory by default.
1204          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1205          *
1206          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1207          * nb_txd can be configured at run time.
1208          */
1209         if (param_total_num_mbufs)
1210                 nb_mbuf_per_pool = param_total_num_mbufs;
1211         else {
1212                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1213                         (nb_lcores * mb_mempool_cache) +
1214                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1215                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1216         }
1217
1218         if (numa_support) {
1219                 uint8_t i;
1220
1221                 for (i = 0; i < num_sockets; i++)
1222                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1223                                                        nb_mbuf_per_pool,
1224                                                        socket_ids[i]);
1225         } else {
1226                 if (socket_num == UMA_NO_CONFIG)
1227                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1228                                                        nb_mbuf_per_pool, 0);
1229                 else
1230                         mempools[socket_num] = mbuf_pool_create
1231                                                         (mbuf_data_size,
1232                                                          nb_mbuf_per_pool,
1233                                                          socket_num);
1234         }
1235
1236         init_port_config();
1237
1238         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1239                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1240         /*
1241          * Records which Mbuf pool to use by each logical core, if needed.
1242          */
1243         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1244                 mbp = mbuf_pool_find(
1245                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1246
1247                 if (mbp == NULL)
1248                         mbp = mbuf_pool_find(0);
1249                 fwd_lcores[lc_id]->mbp = mbp;
1250                 /* initialize GSO context */
1251                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1252                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1253                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1254                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1255                         RTE_ETHER_CRC_LEN;
1256                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1257         }
1258
1259         /* Configuration of packet forwarding streams. */
1260         if (init_fwd_streams() < 0)
1261                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1262
1263         fwd_config_setup();
1264
1265         /* create a gro context for each lcore */
1266         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1267         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1268         gro_param.max_item_per_flow = MAX_PKT_BURST;
1269         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1270                 gro_param.socket_id = rte_lcore_to_socket_id(
1271                                 fwd_lcores_cpuids[lc_id]);
1272                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1273                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1274                         rte_exit(EXIT_FAILURE,
1275                                         "rte_gro_ctx_create() failed\n");
1276                 }
1277         }
1278
1279 #if defined RTE_LIBRTE_PMD_SOFTNIC
1280         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1281                 RTE_ETH_FOREACH_DEV(pid) {
1282                         port = &ports[pid];
1283                         const char *driver = port->dev_info.driver_name;
1284
1285                         if (strcmp(driver, "net_softnic") == 0)
1286                                 port->softport.fwd_lcore_arg = fwd_lcores;
1287                 }
1288         }
1289 #endif
1290
1291 }
1292
1293
1294 void
1295 reconfig(portid_t new_port_id, unsigned socket_id)
1296 {
1297         struct rte_port *port;
1298
1299         /* Reconfiguration of Ethernet ports. */
1300         port = &ports[new_port_id];
1301         rte_eth_dev_info_get(new_port_id, &port->dev_info);
1302
1303         /* set flag to initialize port/queue */
1304         port->need_reconfig = 1;
1305         port->need_reconfig_queues = 1;
1306         port->socket_id = socket_id;
1307
1308         init_port_config();
1309 }
1310
1311
1312 int
1313 init_fwd_streams(void)
1314 {
1315         portid_t pid;
1316         struct rte_port *port;
1317         streamid_t sm_id, nb_fwd_streams_new;
1318         queueid_t q;
1319
1320         /* set socket id according to numa or not */
1321         RTE_ETH_FOREACH_DEV(pid) {
1322                 port = &ports[pid];
1323                 if (nb_rxq > port->dev_info.max_rx_queues) {
1324                         printf("Fail: nb_rxq(%d) is greater than "
1325                                 "max_rx_queues(%d)\n", nb_rxq,
1326                                 port->dev_info.max_rx_queues);
1327                         return -1;
1328                 }
1329                 if (nb_txq > port->dev_info.max_tx_queues) {
1330                         printf("Fail: nb_txq(%d) is greater than "
1331                                 "max_tx_queues(%d)\n", nb_txq,
1332                                 port->dev_info.max_tx_queues);
1333                         return -1;
1334                 }
1335                 if (numa_support) {
1336                         if (port_numa[pid] != NUMA_NO_CONFIG)
1337                                 port->socket_id = port_numa[pid];
1338                         else {
1339                                 port->socket_id = rte_eth_dev_socket_id(pid);
1340
1341                                 /*
1342                                  * if socket_id is invalid,
1343                                  * set to the first available socket.
1344                                  */
1345                                 if (check_socket_id(port->socket_id) < 0)
1346                                         port->socket_id = socket_ids[0];
1347                         }
1348                 }
1349                 else {
1350                         if (socket_num == UMA_NO_CONFIG)
1351                                 port->socket_id = 0;
1352                         else
1353                                 port->socket_id = socket_num;
1354                 }
1355         }
1356
1357         q = RTE_MAX(nb_rxq, nb_txq);
1358         if (q == 0) {
1359                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1360                 return -1;
1361         }
1362         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1363         if (nb_fwd_streams_new == nb_fwd_streams)
1364                 return 0;
1365         /* clear the old */
1366         if (fwd_streams != NULL) {
1367                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1368                         if (fwd_streams[sm_id] == NULL)
1369                                 continue;
1370                         rte_free(fwd_streams[sm_id]);
1371                         fwd_streams[sm_id] = NULL;
1372                 }
1373                 rte_free(fwd_streams);
1374                 fwd_streams = NULL;
1375         }
1376
1377         /* init new */
1378         nb_fwd_streams = nb_fwd_streams_new;
1379         if (nb_fwd_streams) {
1380                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1381                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1382                         RTE_CACHE_LINE_SIZE);
1383                 if (fwd_streams == NULL)
1384                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1385                                  " (struct fwd_stream *)) failed\n",
1386                                  nb_fwd_streams);
1387
1388                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1389                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1390                                 " struct fwd_stream", sizeof(struct fwd_stream),
1391                                 RTE_CACHE_LINE_SIZE);
1392                         if (fwd_streams[sm_id] == NULL)
1393                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1394                                          "(struct fwd_stream) failed\n");
1395                 }
1396         }
1397
1398         return 0;
1399 }
1400
1401 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1402 static void
1403 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1404 {
1405         unsigned int total_burst;
1406         unsigned int nb_burst;
1407         unsigned int burst_stats[3];
1408         uint16_t pktnb_stats[3];
1409         uint16_t nb_pkt;
1410         int burst_percent[3];
1411
1412         /*
1413          * First compute the total number of packet bursts and the
1414          * two highest numbers of bursts of the same number of packets.
1415          */
1416         total_burst = 0;
1417         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1418         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1419         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1420                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1421                 if (nb_burst == 0)
1422                         continue;
1423                 total_burst += nb_burst;
1424                 if (nb_burst > burst_stats[0]) {
1425                         burst_stats[1] = burst_stats[0];
1426                         pktnb_stats[1] = pktnb_stats[0];
1427                         burst_stats[0] = nb_burst;
1428                         pktnb_stats[0] = nb_pkt;
1429                 } else if (nb_burst > burst_stats[1]) {
1430                         burst_stats[1] = nb_burst;
1431                         pktnb_stats[1] = nb_pkt;
1432                 }
1433         }
1434         if (total_burst == 0)
1435                 return;
1436         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1437         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1438                burst_percent[0], (int) pktnb_stats[0]);
1439         if (burst_stats[0] == total_burst) {
1440                 printf("]\n");
1441                 return;
1442         }
1443         if (burst_stats[0] + burst_stats[1] == total_burst) {
1444                 printf(" + %d%% of %d pkts]\n",
1445                        100 - burst_percent[0], pktnb_stats[1]);
1446                 return;
1447         }
1448         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1449         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1450         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1451                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1452                 return;
1453         }
1454         printf(" + %d%% of %d pkts + %d%% of others]\n",
1455                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1456 }
1457 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1458
1459 static void
1460 fwd_stream_stats_display(streamid_t stream_id)
1461 {
1462         struct fwd_stream *fs;
1463         static const char *fwd_top_stats_border = "-------";
1464
1465         fs = fwd_streams[stream_id];
1466         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1467             (fs->fwd_dropped == 0))
1468                 return;
1469         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1470                "TX Port=%2d/Queue=%2d %s\n",
1471                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1472                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1473         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1474                " TX-dropped: %-14"PRIu64,
1475                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1476
1477         /* if checksum mode */
1478         if (cur_fwd_eng == &csum_fwd_engine) {
1479                 printf("  RX- bad IP checksum: %-14"PRIu64
1480                        "  Rx- bad L4 checksum: %-14"PRIu64
1481                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1482                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1483                         fs->rx_bad_outer_l4_csum);
1484         } else {
1485                 printf("\n");
1486         }
1487
1488 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1489         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1490         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1491 #endif
1492 }
1493
1494 void
1495 fwd_stats_display(void)
1496 {
1497         static const char *fwd_stats_border = "----------------------";
1498         static const char *acc_stats_border = "+++++++++++++++";
1499         struct {
1500                 struct fwd_stream *rx_stream;
1501                 struct fwd_stream *tx_stream;
1502                 uint64_t tx_dropped;
1503                 uint64_t rx_bad_ip_csum;
1504                 uint64_t rx_bad_l4_csum;
1505                 uint64_t rx_bad_outer_l4_csum;
1506         } ports_stats[RTE_MAX_ETHPORTS];
1507         uint64_t total_rx_dropped = 0;
1508         uint64_t total_tx_dropped = 0;
1509         uint64_t total_rx_nombuf = 0;
1510         struct rte_eth_stats stats;
1511 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1512         uint64_t fwd_cycles = 0;
1513 #endif
1514         uint64_t total_recv = 0;
1515         uint64_t total_xmit = 0;
1516         struct rte_port *port;
1517         streamid_t sm_id;
1518         portid_t pt_id;
1519         int i;
1520
1521         memset(ports_stats, 0, sizeof(ports_stats));
1522
1523         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1524                 struct fwd_stream *fs = fwd_streams[sm_id];
1525
1526                 if (cur_fwd_config.nb_fwd_streams >
1527                     cur_fwd_config.nb_fwd_ports) {
1528                         fwd_stream_stats_display(sm_id);
1529                 } else {
1530                         ports_stats[fs->tx_port].tx_stream = fs;
1531                         ports_stats[fs->rx_port].rx_stream = fs;
1532                 }
1533
1534                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1535
1536                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1537                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1538                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1539                                 fs->rx_bad_outer_l4_csum;
1540
1541 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1542                 fwd_cycles += fs->core_cycles;
1543 #endif
1544         }
1545         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1546                 uint8_t j;
1547
1548                 pt_id = fwd_ports_ids[i];
1549                 port = &ports[pt_id];
1550
1551                 rte_eth_stats_get(pt_id, &stats);
1552                 stats.ipackets -= port->stats.ipackets;
1553                 stats.opackets -= port->stats.opackets;
1554                 stats.ibytes -= port->stats.ibytes;
1555                 stats.obytes -= port->stats.obytes;
1556                 stats.imissed -= port->stats.imissed;
1557                 stats.oerrors -= port->stats.oerrors;
1558                 stats.rx_nombuf -= port->stats.rx_nombuf;
1559
1560                 total_recv += stats.ipackets;
1561                 total_xmit += stats.opackets;
1562                 total_rx_dropped += stats.imissed;
1563                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1564                 total_tx_dropped += stats.oerrors;
1565                 total_rx_nombuf  += stats.rx_nombuf;
1566
1567                 printf("\n  %s Forward statistics for port %-2d %s\n",
1568                        fwd_stats_border, pt_id, fwd_stats_border);
1569
1570                 if (!port->rx_queue_stats_mapping_enabled &&
1571                     !port->tx_queue_stats_mapping_enabled) {
1572                         printf("  RX-packets: %-14"PRIu64
1573                                " RX-dropped: %-14"PRIu64
1574                                "RX-total: %-"PRIu64"\n",
1575                                stats.ipackets, stats.imissed,
1576                                stats.ipackets + stats.imissed);
1577
1578                         if (cur_fwd_eng == &csum_fwd_engine)
1579                                 printf("  Bad-ipcsum: %-14"PRIu64
1580                                        " Bad-l4csum: %-14"PRIu64
1581                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1582                                        ports_stats[pt_id].rx_bad_ip_csum,
1583                                        ports_stats[pt_id].rx_bad_l4_csum,
1584                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1585                         if (stats.ierrors + stats.rx_nombuf > 0) {
1586                                 printf("  RX-error: %-"PRIu64"\n",
1587                                        stats.ierrors);
1588                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1589                                        stats.rx_nombuf);
1590                         }
1591
1592                         printf("  TX-packets: %-14"PRIu64
1593                                " TX-dropped: %-14"PRIu64
1594                                "TX-total: %-"PRIu64"\n",
1595                                stats.opackets, ports_stats[pt_id].tx_dropped,
1596                                stats.opackets + ports_stats[pt_id].tx_dropped);
1597                 } else {
1598                         printf("  RX-packets:             %14"PRIu64
1599                                "    RX-dropped:%14"PRIu64
1600                                "    RX-total:%14"PRIu64"\n",
1601                                stats.ipackets, stats.imissed,
1602                                stats.ipackets + stats.imissed);
1603
1604                         if (cur_fwd_eng == &csum_fwd_engine)
1605                                 printf("  Bad-ipcsum:%14"PRIu64
1606                                        "    Bad-l4csum:%14"PRIu64
1607                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1608                                        ports_stats[pt_id].rx_bad_ip_csum,
1609                                        ports_stats[pt_id].rx_bad_l4_csum,
1610                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1611                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1612                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1613                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1614                                        stats.rx_nombuf);
1615                         }
1616
1617                         printf("  TX-packets:             %14"PRIu64
1618                                "    TX-dropped:%14"PRIu64
1619                                "    TX-total:%14"PRIu64"\n",
1620                                stats.opackets, ports_stats[pt_id].tx_dropped,
1621                                stats.opackets + ports_stats[pt_id].tx_dropped);
1622                 }
1623
1624 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1625                 if (ports_stats[pt_id].rx_stream)
1626                         pkt_burst_stats_display("RX",
1627                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1628                 if (ports_stats[pt_id].tx_stream)
1629                         pkt_burst_stats_display("TX",
1630                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1631 #endif
1632
1633                 if (port->rx_queue_stats_mapping_enabled) {
1634                         printf("\n");
1635                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1636                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1637                                        "     RX-errors:%14"PRIu64
1638                                        "    RX-bytes:%14"PRIu64"\n",
1639                                        j, stats.q_ipackets[j],
1640                                        stats.q_errors[j], stats.q_ibytes[j]);
1641                         }
1642                         printf("\n");
1643                 }
1644                 if (port->tx_queue_stats_mapping_enabled) {
1645                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1646                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1647                                        "                                 TX-bytes:%14"
1648                                        PRIu64"\n",
1649                                        j, stats.q_opackets[j],
1650                                        stats.q_obytes[j]);
1651                         }
1652                 }
1653
1654                 printf("  %s--------------------------------%s\n",
1655                        fwd_stats_border, fwd_stats_border);
1656         }
1657
1658         printf("\n  %s Accumulated forward statistics for all ports"
1659                "%s\n",
1660                acc_stats_border, acc_stats_border);
1661         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1662                "%-"PRIu64"\n"
1663                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1664                "%-"PRIu64"\n",
1665                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1666                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1667         if (total_rx_nombuf > 0)
1668                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1669         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1670                "%s\n",
1671                acc_stats_border, acc_stats_border);
1672 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1673         if (total_recv > 0)
1674                 printf("\n  CPU cycles/packet=%u (total cycles="
1675                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1676                        (unsigned int)(fwd_cycles / total_recv),
1677                        fwd_cycles, total_recv);
1678 #endif
1679 }
1680
1681 void
1682 fwd_stats_reset(void)
1683 {
1684         streamid_t sm_id;
1685         portid_t pt_id;
1686         int i;
1687
1688         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1689                 pt_id = fwd_ports_ids[i];
1690                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1691         }
1692         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1693                 struct fwd_stream *fs = fwd_streams[sm_id];
1694
1695                 fs->rx_packets = 0;
1696                 fs->tx_packets = 0;
1697                 fs->fwd_dropped = 0;
1698                 fs->rx_bad_ip_csum = 0;
1699                 fs->rx_bad_l4_csum = 0;
1700                 fs->rx_bad_outer_l4_csum = 0;
1701
1702 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1703                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1704                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1705 #endif
1706 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1707                 fs->core_cycles = 0;
1708 #endif
1709         }
1710 }
1711
1712 static void
1713 flush_fwd_rx_queues(void)
1714 {
1715         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1716         portid_t  rxp;
1717         portid_t port_id;
1718         queueid_t rxq;
1719         uint16_t  nb_rx;
1720         uint16_t  i;
1721         uint8_t   j;
1722         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1723         uint64_t timer_period;
1724
1725         /* convert to number of cycles */
1726         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1727
1728         for (j = 0; j < 2; j++) {
1729                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1730                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1731                                 port_id = fwd_ports_ids[rxp];
1732                                 /**
1733                                 * testpmd can stuck in the below do while loop
1734                                 * if rte_eth_rx_burst() always returns nonzero
1735                                 * packets. So timer is added to exit this loop
1736                                 * after 1sec timer expiry.
1737                                 */
1738                                 prev_tsc = rte_rdtsc();
1739                                 do {
1740                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1741                                                 pkts_burst, MAX_PKT_BURST);
1742                                         for (i = 0; i < nb_rx; i++)
1743                                                 rte_pktmbuf_free(pkts_burst[i]);
1744
1745                                         cur_tsc = rte_rdtsc();
1746                                         diff_tsc = cur_tsc - prev_tsc;
1747                                         timer_tsc += diff_tsc;
1748                                 } while ((nb_rx > 0) &&
1749                                         (timer_tsc < timer_period));
1750                                 timer_tsc = 0;
1751                         }
1752                 }
1753                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1754         }
1755 }
1756
1757 static void
1758 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1759 {
1760         struct fwd_stream **fsm;
1761         streamid_t nb_fs;
1762         streamid_t sm_id;
1763 #ifdef RTE_LIBRTE_BITRATE
1764         uint64_t tics_per_1sec;
1765         uint64_t tics_datum;
1766         uint64_t tics_current;
1767         uint16_t i, cnt_ports;
1768
1769         cnt_ports = nb_ports;
1770         tics_datum = rte_rdtsc();
1771         tics_per_1sec = rte_get_timer_hz();
1772 #endif
1773         fsm = &fwd_streams[fc->stream_idx];
1774         nb_fs = fc->stream_nb;
1775         do {
1776                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1777                         (*pkt_fwd)(fsm[sm_id]);
1778 #ifdef RTE_LIBRTE_BITRATE
1779                 if (bitrate_enabled != 0 &&
1780                                 bitrate_lcore_id == rte_lcore_id()) {
1781                         tics_current = rte_rdtsc();
1782                         if (tics_current - tics_datum >= tics_per_1sec) {
1783                                 /* Periodic bitrate calculation */
1784                                 for (i = 0; i < cnt_ports; i++)
1785                                         rte_stats_bitrate_calc(bitrate_data,
1786                                                 ports_ids[i]);
1787                                 tics_datum = tics_current;
1788                         }
1789                 }
1790 #endif
1791 #ifdef RTE_LIBRTE_LATENCY_STATS
1792                 if (latencystats_enabled != 0 &&
1793                                 latencystats_lcore_id == rte_lcore_id())
1794                         rte_latencystats_update();
1795 #endif
1796
1797         } while (! fc->stopped);
1798 }
1799
1800 static int
1801 start_pkt_forward_on_core(void *fwd_arg)
1802 {
1803         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1804                              cur_fwd_config.fwd_eng->packet_fwd);
1805         return 0;
1806 }
1807
1808 /*
1809  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1810  * Used to start communication flows in network loopback test configurations.
1811  */
1812 static int
1813 run_one_txonly_burst_on_core(void *fwd_arg)
1814 {
1815         struct fwd_lcore *fwd_lc;
1816         struct fwd_lcore tmp_lcore;
1817
1818         fwd_lc = (struct fwd_lcore *) fwd_arg;
1819         tmp_lcore = *fwd_lc;
1820         tmp_lcore.stopped = 1;
1821         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1822         return 0;
1823 }
1824
1825 /*
1826  * Launch packet forwarding:
1827  *     - Setup per-port forwarding context.
1828  *     - launch logical cores with their forwarding configuration.
1829  */
1830 static void
1831 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1832 {
1833         port_fwd_begin_t port_fwd_begin;
1834         unsigned int i;
1835         unsigned int lc_id;
1836         int diag;
1837
1838         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1839         if (port_fwd_begin != NULL) {
1840                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1841                         (*port_fwd_begin)(fwd_ports_ids[i]);
1842         }
1843         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1844                 lc_id = fwd_lcores_cpuids[i];
1845                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1846                         fwd_lcores[i]->stopped = 0;
1847                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1848                                                      fwd_lcores[i], lc_id);
1849                         if (diag != 0)
1850                                 printf("launch lcore %u failed - diag=%d\n",
1851                                        lc_id, diag);
1852                 }
1853         }
1854 }
1855
1856 /*
1857  * Launch packet forwarding configuration.
1858  */
1859 void
1860 start_packet_forwarding(int with_tx_first)
1861 {
1862         port_fwd_begin_t port_fwd_begin;
1863         port_fwd_end_t  port_fwd_end;
1864         struct rte_port *port;
1865         unsigned int i;
1866         portid_t   pt_id;
1867
1868         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1869                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1870
1871         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1872                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1873
1874         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1875                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1876                 (!nb_rxq || !nb_txq))
1877                 rte_exit(EXIT_FAILURE,
1878                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1879                         cur_fwd_eng->fwd_mode_name);
1880
1881         if (all_ports_started() == 0) {
1882                 printf("Not all ports were started\n");
1883                 return;
1884         }
1885         if (test_done == 0) {
1886                 printf("Packet forwarding already started\n");
1887                 return;
1888         }
1889
1890
1891         if(dcb_test) {
1892                 for (i = 0; i < nb_fwd_ports; i++) {
1893                         pt_id = fwd_ports_ids[i];
1894                         port = &ports[pt_id];
1895                         if (!port->dcb_flag) {
1896                                 printf("In DCB mode, all forwarding ports must "
1897                                        "be configured in this mode.\n");
1898                                 return;
1899                         }
1900                 }
1901                 if (nb_fwd_lcores == 1) {
1902                         printf("In DCB mode,the nb forwarding cores "
1903                                "should be larger than 1.\n");
1904                         return;
1905                 }
1906         }
1907         test_done = 0;
1908
1909         fwd_config_setup();
1910
1911         if(!no_flush_rx)
1912                 flush_fwd_rx_queues();
1913
1914         pkt_fwd_config_display(&cur_fwd_config);
1915         rxtx_config_display();
1916
1917         fwd_stats_reset();
1918         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1919                 pt_id = fwd_ports_ids[i];
1920                 port = &ports[pt_id];
1921                 map_port_queue_stats_mapping_registers(pt_id, port);
1922         }
1923         if (with_tx_first) {
1924                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1925                 if (port_fwd_begin != NULL) {
1926                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1927                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1928                 }
1929                 while (with_tx_first--) {
1930                         launch_packet_forwarding(
1931                                         run_one_txonly_burst_on_core);
1932                         rte_eal_mp_wait_lcore();
1933                 }
1934                 port_fwd_end = tx_only_engine.port_fwd_end;
1935                 if (port_fwd_end != NULL) {
1936                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1937                                 (*port_fwd_end)(fwd_ports_ids[i]);
1938                 }
1939         }
1940         launch_packet_forwarding(start_pkt_forward_on_core);
1941 }
1942
1943 void
1944 stop_packet_forwarding(void)
1945 {
1946         port_fwd_end_t port_fwd_end;
1947         lcoreid_t lc_id;
1948         portid_t pt_id;
1949         int i;
1950
1951         if (test_done) {
1952                 printf("Packet forwarding not started\n");
1953                 return;
1954         }
1955         printf("Telling cores to stop...");
1956         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1957                 fwd_lcores[lc_id]->stopped = 1;
1958         printf("\nWaiting for lcores to finish...\n");
1959         rte_eal_mp_wait_lcore();
1960         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1961         if (port_fwd_end != NULL) {
1962                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1963                         pt_id = fwd_ports_ids[i];
1964                         (*port_fwd_end)(pt_id);
1965                 }
1966         }
1967
1968         fwd_stats_display();
1969
1970         printf("\nDone.\n");
1971         test_done = 1;
1972 }
1973
1974 void
1975 dev_set_link_up(portid_t pid)
1976 {
1977         if (rte_eth_dev_set_link_up(pid) < 0)
1978                 printf("\nSet link up fail.\n");
1979 }
1980
1981 void
1982 dev_set_link_down(portid_t pid)
1983 {
1984         if (rte_eth_dev_set_link_down(pid) < 0)
1985                 printf("\nSet link down fail.\n");
1986 }
1987
1988 static int
1989 all_ports_started(void)
1990 {
1991         portid_t pi;
1992         struct rte_port *port;
1993
1994         RTE_ETH_FOREACH_DEV(pi) {
1995                 port = &ports[pi];
1996                 /* Check if there is a port which is not started */
1997                 if ((port->port_status != RTE_PORT_STARTED) &&
1998                         (port->slave_flag == 0))
1999                         return 0;
2000         }
2001
2002         /* No port is not started */
2003         return 1;
2004 }
2005
2006 int
2007 port_is_stopped(portid_t port_id)
2008 {
2009         struct rte_port *port = &ports[port_id];
2010
2011         if ((port->port_status != RTE_PORT_STOPPED) &&
2012             (port->slave_flag == 0))
2013                 return 0;
2014         return 1;
2015 }
2016
2017 int
2018 all_ports_stopped(void)
2019 {
2020         portid_t pi;
2021
2022         RTE_ETH_FOREACH_DEV(pi) {
2023                 if (!port_is_stopped(pi))
2024                         return 0;
2025         }
2026
2027         return 1;
2028 }
2029
2030 int
2031 port_is_started(portid_t port_id)
2032 {
2033         if (port_id_is_invalid(port_id, ENABLED_WARN))
2034                 return 0;
2035
2036         if (ports[port_id].port_status != RTE_PORT_STARTED)
2037                 return 0;
2038
2039         return 1;
2040 }
2041
2042 int
2043 start_port(portid_t pid)
2044 {
2045         int diag, need_check_link_status = -1;
2046         portid_t pi;
2047         queueid_t qi;
2048         struct rte_port *port;
2049         struct rte_ether_addr mac_addr;
2050
2051         if (port_id_is_invalid(pid, ENABLED_WARN))
2052                 return 0;
2053
2054         if(dcb_config)
2055                 dcb_test = 1;
2056         RTE_ETH_FOREACH_DEV(pi) {
2057                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2058                         continue;
2059
2060                 need_check_link_status = 0;
2061                 port = &ports[pi];
2062                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2063                                                  RTE_PORT_HANDLING) == 0) {
2064                         printf("Port %d is now not stopped\n", pi);
2065                         continue;
2066                 }
2067
2068                 if (port->need_reconfig > 0) {
2069                         port->need_reconfig = 0;
2070
2071                         if (flow_isolate_all) {
2072                                 int ret = port_flow_isolate(pi, 1);
2073                                 if (ret) {
2074                                         printf("Failed to apply isolated"
2075                                                " mode on port %d\n", pi);
2076                                         return -1;
2077                                 }
2078                         }
2079                         configure_rxtx_dump_callbacks(0);
2080                         printf("Configuring Port %d (socket %u)\n", pi,
2081                                         port->socket_id);
2082                         /* configure port */
2083                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
2084                                                 &(port->dev_conf));
2085                         if (diag != 0) {
2086                                 if (rte_atomic16_cmpset(&(port->port_status),
2087                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2088                                         printf("Port %d can not be set back "
2089                                                         "to stopped\n", pi);
2090                                 printf("Fail to configure port %d\n", pi);
2091                                 /* try to reconfigure port next time */
2092                                 port->need_reconfig = 1;
2093                                 return -1;
2094                         }
2095                 }
2096                 if (port->need_reconfig_queues > 0) {
2097                         port->need_reconfig_queues = 0;
2098                         /* setup tx queues */
2099                         for (qi = 0; qi < nb_txq; qi++) {
2100                                 if ((numa_support) &&
2101                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2102                                         diag = rte_eth_tx_queue_setup(pi, qi,
2103                                                 port->nb_tx_desc[qi],
2104                                                 txring_numa[pi],
2105                                                 &(port->tx_conf[qi]));
2106                                 else
2107                                         diag = rte_eth_tx_queue_setup(pi, qi,
2108                                                 port->nb_tx_desc[qi],
2109                                                 port->socket_id,
2110                                                 &(port->tx_conf[qi]));
2111
2112                                 if (diag == 0)
2113                                         continue;
2114
2115                                 /* Fail to setup tx queue, return */
2116                                 if (rte_atomic16_cmpset(&(port->port_status),
2117                                                         RTE_PORT_HANDLING,
2118                                                         RTE_PORT_STOPPED) == 0)
2119                                         printf("Port %d can not be set back "
2120                                                         "to stopped\n", pi);
2121                                 printf("Fail to configure port %d tx queues\n",
2122                                        pi);
2123                                 /* try to reconfigure queues next time */
2124                                 port->need_reconfig_queues = 1;
2125                                 return -1;
2126                         }
2127                         for (qi = 0; qi < nb_rxq; qi++) {
2128                                 /* setup rx queues */
2129                                 if ((numa_support) &&
2130                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2131                                         struct rte_mempool * mp =
2132                                                 mbuf_pool_find(rxring_numa[pi]);
2133                                         if (mp == NULL) {
2134                                                 printf("Failed to setup RX queue:"
2135                                                         "No mempool allocation"
2136                                                         " on the socket %d\n",
2137                                                         rxring_numa[pi]);
2138                                                 return -1;
2139                                         }
2140
2141                                         diag = rte_eth_rx_queue_setup(pi, qi,
2142                                              port->nb_rx_desc[qi],
2143                                              rxring_numa[pi],
2144                                              &(port->rx_conf[qi]),
2145                                              mp);
2146                                 } else {
2147                                         struct rte_mempool *mp =
2148                                                 mbuf_pool_find(port->socket_id);
2149                                         if (mp == NULL) {
2150                                                 printf("Failed to setup RX queue:"
2151                                                         "No mempool allocation"
2152                                                         " on the socket %d\n",
2153                                                         port->socket_id);
2154                                                 return -1;
2155                                         }
2156                                         diag = rte_eth_rx_queue_setup(pi, qi,
2157                                              port->nb_rx_desc[qi],
2158                                              port->socket_id,
2159                                              &(port->rx_conf[qi]),
2160                                              mp);
2161                                 }
2162                                 if (diag == 0)
2163                                         continue;
2164
2165                                 /* Fail to setup rx queue, return */
2166                                 if (rte_atomic16_cmpset(&(port->port_status),
2167                                                         RTE_PORT_HANDLING,
2168                                                         RTE_PORT_STOPPED) == 0)
2169                                         printf("Port %d can not be set back "
2170                                                         "to stopped\n", pi);
2171                                 printf("Fail to configure port %d rx queues\n",
2172                                        pi);
2173                                 /* try to reconfigure queues next time */
2174                                 port->need_reconfig_queues = 1;
2175                                 return -1;
2176                         }
2177                 }
2178                 configure_rxtx_dump_callbacks(verbose_level);
2179                 /* start port */
2180                 if (rte_eth_dev_start(pi) < 0) {
2181                         printf("Fail to start port %d\n", pi);
2182
2183                         /* Fail to setup rx queue, return */
2184                         if (rte_atomic16_cmpset(&(port->port_status),
2185                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2186                                 printf("Port %d can not be set back to "
2187                                                         "stopped\n", pi);
2188                         continue;
2189                 }
2190
2191                 if (rte_atomic16_cmpset(&(port->port_status),
2192                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2193                         printf("Port %d can not be set into started\n", pi);
2194
2195                 rte_eth_macaddr_get(pi, &mac_addr);
2196                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2197                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2198                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2199                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2200
2201                 /* at least one port started, need checking link status */
2202                 need_check_link_status = 1;
2203         }
2204
2205         if (need_check_link_status == 1 && !no_link_check)
2206                 check_all_ports_link_status(RTE_PORT_ALL);
2207         else if (need_check_link_status == 0)
2208                 printf("Please stop the ports first\n");
2209
2210         printf("Done\n");
2211         return 0;
2212 }
2213
2214 void
2215 stop_port(portid_t pid)
2216 {
2217         portid_t pi;
2218         struct rte_port *port;
2219         int need_check_link_status = 0;
2220
2221         if (dcb_test) {
2222                 dcb_test = 0;
2223                 dcb_config = 0;
2224         }
2225
2226         if (port_id_is_invalid(pid, ENABLED_WARN))
2227                 return;
2228
2229         printf("Stopping ports...\n");
2230
2231         RTE_ETH_FOREACH_DEV(pi) {
2232                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2233                         continue;
2234
2235                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2236                         printf("Please remove port %d from forwarding configuration.\n", pi);
2237                         continue;
2238                 }
2239
2240                 if (port_is_bonding_slave(pi)) {
2241                         printf("Please remove port %d from bonded device.\n", pi);
2242                         continue;
2243                 }
2244
2245                 port = &ports[pi];
2246                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2247                                                 RTE_PORT_HANDLING) == 0)
2248                         continue;
2249
2250                 rte_eth_dev_stop(pi);
2251
2252                 if (rte_atomic16_cmpset(&(port->port_status),
2253                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2254                         printf("Port %d can not be set into stopped\n", pi);
2255                 need_check_link_status = 1;
2256         }
2257         if (need_check_link_status && !no_link_check)
2258                 check_all_ports_link_status(RTE_PORT_ALL);
2259
2260         printf("Done\n");
2261 }
2262
2263 static void
2264 remove_invalid_ports_in(portid_t *array, portid_t *total)
2265 {
2266         portid_t i;
2267         portid_t new_total = 0;
2268
2269         for (i = 0; i < *total; i++)
2270                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2271                         array[new_total] = array[i];
2272                         new_total++;
2273                 }
2274         *total = new_total;
2275 }
2276
2277 static void
2278 remove_invalid_ports(void)
2279 {
2280         remove_invalid_ports_in(ports_ids, &nb_ports);
2281         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2282         nb_cfg_ports = nb_fwd_ports;
2283 }
2284
2285 void
2286 close_port(portid_t pid)
2287 {
2288         portid_t pi;
2289         struct rte_port *port;
2290
2291         if (port_id_is_invalid(pid, ENABLED_WARN))
2292                 return;
2293
2294         printf("Closing ports...\n");
2295
2296         RTE_ETH_FOREACH_DEV(pi) {
2297                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2298                         continue;
2299
2300                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2301                         printf("Please remove port %d from forwarding configuration.\n", pi);
2302                         continue;
2303                 }
2304
2305                 if (port_is_bonding_slave(pi)) {
2306                         printf("Please remove port %d from bonded device.\n", pi);
2307                         continue;
2308                 }
2309
2310                 port = &ports[pi];
2311                 if (rte_atomic16_cmpset(&(port->port_status),
2312                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2313                         printf("Port %d is already closed\n", pi);
2314                         continue;
2315                 }
2316
2317                 if (rte_atomic16_cmpset(&(port->port_status),
2318                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2319                         printf("Port %d is now not stopped\n", pi);
2320                         continue;
2321                 }
2322
2323                 if (port->flow_list)
2324                         port_flow_flush(pi);
2325                 rte_eth_dev_close(pi);
2326
2327                 remove_invalid_ports();
2328
2329                 if (rte_atomic16_cmpset(&(port->port_status),
2330                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2331                         printf("Port %d cannot be set to closed\n", pi);
2332         }
2333
2334         printf("Done\n");
2335 }
2336
2337 void
2338 reset_port(portid_t pid)
2339 {
2340         int diag;
2341         portid_t pi;
2342         struct rte_port *port;
2343
2344         if (port_id_is_invalid(pid, ENABLED_WARN))
2345                 return;
2346
2347         printf("Resetting ports...\n");
2348
2349         RTE_ETH_FOREACH_DEV(pi) {
2350                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2351                         continue;
2352
2353                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2354                         printf("Please remove port %d from forwarding "
2355                                "configuration.\n", pi);
2356                         continue;
2357                 }
2358
2359                 if (port_is_bonding_slave(pi)) {
2360                         printf("Please remove port %d from bonded device.\n",
2361                                pi);
2362                         continue;
2363                 }
2364
2365                 diag = rte_eth_dev_reset(pi);
2366                 if (diag == 0) {
2367                         port = &ports[pi];
2368                         port->need_reconfig = 1;
2369                         port->need_reconfig_queues = 1;
2370                 } else {
2371                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2372                 }
2373         }
2374
2375         printf("Done\n");
2376 }
2377
2378 void
2379 attach_port(char *identifier)
2380 {
2381         portid_t pi;
2382         struct rte_dev_iterator iterator;
2383
2384         printf("Attaching a new port...\n");
2385
2386         if (identifier == NULL) {
2387                 printf("Invalid parameters are specified\n");
2388                 return;
2389         }
2390
2391         if (rte_dev_probe(identifier) < 0) {
2392                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2393                 return;
2394         }
2395
2396         /* first attach mode: event */
2397         if (setup_on_probe_event) {
2398                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2399                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2400                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2401                                         ports[pi].need_setup != 0)
2402                                 setup_attached_port(pi);
2403                 return;
2404         }
2405
2406         /* second attach mode: iterator */
2407         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2408                 /* setup ports matching the devargs used for probing */
2409                 if (port_is_forwarding(pi))
2410                         continue; /* port was already attached before */
2411                 setup_attached_port(pi);
2412         }
2413 }
2414
2415 static void
2416 setup_attached_port(portid_t pi)
2417 {
2418         unsigned int socket_id;
2419
2420         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2421         /* if socket_id is invalid, set to the first available socket. */
2422         if (check_socket_id(socket_id) < 0)
2423                 socket_id = socket_ids[0];
2424         reconfig(pi, socket_id);
2425         rte_eth_promiscuous_enable(pi);
2426
2427         ports_ids[nb_ports++] = pi;
2428         fwd_ports_ids[nb_fwd_ports++] = pi;
2429         nb_cfg_ports = nb_fwd_ports;
2430         ports[pi].need_setup = 0;
2431         ports[pi].port_status = RTE_PORT_STOPPED;
2432
2433         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2434         printf("Done\n");
2435 }
2436
2437 void
2438 detach_port_device(portid_t port_id)
2439 {
2440         struct rte_device *dev;
2441         portid_t sibling;
2442
2443         printf("Removing a device...\n");
2444
2445         dev = rte_eth_devices[port_id].device;
2446         if (dev == NULL) {
2447                 printf("Device already removed\n");
2448                 return;
2449         }
2450
2451         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2452                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2453                         printf("Port not stopped\n");
2454                         return;
2455                 }
2456                 printf("Port was not closed\n");
2457                 if (ports[port_id].flow_list)
2458                         port_flow_flush(port_id);
2459         }
2460
2461         if (rte_dev_remove(dev) < 0) {
2462                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2463                 return;
2464         }
2465         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2466                 /* reset mapping between old ports and removed device */
2467                 rte_eth_devices[sibling].device = NULL;
2468                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2469                         /* sibling ports are forced to be closed */
2470                         ports[sibling].port_status = RTE_PORT_CLOSED;
2471                         printf("Port %u is closed\n", sibling);
2472                 }
2473         }
2474
2475         remove_invalid_ports();
2476
2477         printf("Device of port %u is detached\n", port_id);
2478         printf("Now total ports is %d\n", nb_ports);
2479         printf("Done\n");
2480         return;
2481 }
2482
2483 void
2484 detach_device(char *identifier)
2485 {
2486         struct rte_dev_iterator iterator;
2487         struct rte_devargs da;
2488         portid_t port_id;
2489
2490         printf("Removing a device...\n");
2491
2492         memset(&da, 0, sizeof(da));
2493         if (rte_devargs_parsef(&da, "%s", identifier)) {
2494                 printf("cannot parse identifier\n");
2495                 if (da.args)
2496                         free(da.args);
2497                 return;
2498         }
2499
2500         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2501                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2502                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2503                                 printf("Port %u not stopped\n", port_id);
2504                                 return;
2505                         }
2506
2507                         /* sibling ports are forced to be closed */
2508                         if (ports[port_id].flow_list)
2509                                 port_flow_flush(port_id);
2510                         ports[port_id].port_status = RTE_PORT_CLOSED;
2511                         printf("Port %u is now closed\n", port_id);
2512                 }
2513         }
2514
2515         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2516                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2517                             da.name, da.bus->name);
2518                 return;
2519         }
2520
2521         remove_invalid_ports();
2522
2523         printf("Device %s is detached\n", identifier);
2524         printf("Now total ports is %d\n", nb_ports);
2525         printf("Done\n");
2526 }
2527
2528 void
2529 pmd_test_exit(void)
2530 {
2531         portid_t pt_id;
2532         int ret;
2533         int i;
2534
2535         if (test_done == 0)
2536                 stop_packet_forwarding();
2537
2538         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2539                 if (mempools[i]) {
2540                         if (mp_alloc_type == MP_ALLOC_ANON)
2541                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2542                                                      NULL);
2543                 }
2544         }
2545         if (ports != NULL) {
2546                 no_link_check = 1;
2547                 RTE_ETH_FOREACH_DEV(pt_id) {
2548                         printf("\nStopping port %d...\n", pt_id);
2549                         fflush(stdout);
2550                         stop_port(pt_id);
2551                 }
2552                 RTE_ETH_FOREACH_DEV(pt_id) {
2553                         printf("\nShutting down port %d...\n", pt_id);
2554                         fflush(stdout);
2555                         close_port(pt_id);
2556                 }
2557         }
2558
2559         if (hot_plug) {
2560                 ret = rte_dev_event_monitor_stop();
2561                 if (ret) {
2562                         RTE_LOG(ERR, EAL,
2563                                 "fail to stop device event monitor.");
2564                         return;
2565                 }
2566
2567                 ret = rte_dev_event_callback_unregister(NULL,
2568                         dev_event_callback, NULL);
2569                 if (ret < 0) {
2570                         RTE_LOG(ERR, EAL,
2571                                 "fail to unregister device event callback.\n");
2572                         return;
2573                 }
2574
2575                 ret = rte_dev_hotplug_handle_disable();
2576                 if (ret) {
2577                         RTE_LOG(ERR, EAL,
2578                                 "fail to disable hotplug handling.\n");
2579                         return;
2580                 }
2581         }
2582         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2583                 if (mempools[i])
2584                         rte_mempool_free(mempools[i]);
2585         }
2586
2587         printf("\nBye...\n");
2588 }
2589
2590 typedef void (*cmd_func_t)(void);
2591 struct pmd_test_command {
2592         const char *cmd_name;
2593         cmd_func_t cmd_func;
2594 };
2595
2596 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2597
2598 /* Check the link status of all ports in up to 9s, and print them finally */
2599 static void
2600 check_all_ports_link_status(uint32_t port_mask)
2601 {
2602 #define CHECK_INTERVAL 100 /* 100ms */
2603 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2604         portid_t portid;
2605         uint8_t count, all_ports_up, print_flag = 0;
2606         struct rte_eth_link link;
2607
2608         printf("Checking link statuses...\n");
2609         fflush(stdout);
2610         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2611                 all_ports_up = 1;
2612                 RTE_ETH_FOREACH_DEV(portid) {
2613                         if ((port_mask & (1 << portid)) == 0)
2614                                 continue;
2615                         memset(&link, 0, sizeof(link));
2616                         rte_eth_link_get_nowait(portid, &link);
2617                         /* print link status if flag set */
2618                         if (print_flag == 1) {
2619                                 if (link.link_status)
2620                                         printf(
2621                                         "Port%d Link Up. speed %u Mbps- %s\n",
2622                                         portid, link.link_speed,
2623                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2624                                         ("full-duplex") : ("half-duplex\n"));
2625                                 else
2626                                         printf("Port %d Link Down\n", portid);
2627                                 continue;
2628                         }
2629                         /* clear all_ports_up flag if any link down */
2630                         if (link.link_status == ETH_LINK_DOWN) {
2631                                 all_ports_up = 0;
2632                                 break;
2633                         }
2634                 }
2635                 /* after finally printing all link status, get out */
2636                 if (print_flag == 1)
2637                         break;
2638
2639                 if (all_ports_up == 0) {
2640                         fflush(stdout);
2641                         rte_delay_ms(CHECK_INTERVAL);
2642                 }
2643
2644                 /* set the print_flag if all ports up or timeout */
2645                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2646                         print_flag = 1;
2647                 }
2648
2649                 if (lsc_interrupt)
2650                         break;
2651         }
2652 }
2653
2654 /*
2655  * This callback is for remove a port for a device. It has limitation because
2656  * it is not for multiple port removal for a device.
2657  * TODO: the device detach invoke will plan to be removed from user side to
2658  * eal. And convert all PMDs to free port resources on ether device closing.
2659  */
2660 static void
2661 rmv_port_callback(void *arg)
2662 {
2663         int need_to_start = 0;
2664         int org_no_link_check = no_link_check;
2665         portid_t port_id = (intptr_t)arg;
2666
2667         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2668
2669         if (!test_done && port_is_forwarding(port_id)) {
2670                 need_to_start = 1;
2671                 stop_packet_forwarding();
2672         }
2673         no_link_check = 1;
2674         stop_port(port_id);
2675         no_link_check = org_no_link_check;
2676         close_port(port_id);
2677         detach_port_device(port_id);
2678         if (need_to_start)
2679                 start_packet_forwarding(0);
2680 }
2681
2682 /* This function is used by the interrupt thread */
2683 static int
2684 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2685                   void *ret_param)
2686 {
2687         RTE_SET_USED(param);
2688         RTE_SET_USED(ret_param);
2689
2690         if (type >= RTE_ETH_EVENT_MAX) {
2691                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2692                         port_id, __func__, type);
2693                 fflush(stderr);
2694         } else if (event_print_mask & (UINT32_C(1) << type)) {
2695                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2696                         eth_event_desc[type]);
2697                 fflush(stdout);
2698         }
2699
2700         switch (type) {
2701         case RTE_ETH_EVENT_NEW:
2702                 ports[port_id].need_setup = 1;
2703                 ports[port_id].port_status = RTE_PORT_HANDLING;
2704                 break;
2705         case RTE_ETH_EVENT_INTR_RMV:
2706                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2707                         break;
2708                 if (rte_eal_alarm_set(100000,
2709                                 rmv_port_callback, (void *)(intptr_t)port_id))
2710                         fprintf(stderr, "Could not set up deferred device removal\n");
2711                 break;
2712         default:
2713                 break;
2714         }
2715         return 0;
2716 }
2717
2718 static int
2719 register_eth_event_callback(void)
2720 {
2721         int ret;
2722         enum rte_eth_event_type event;
2723
2724         for (event = RTE_ETH_EVENT_UNKNOWN;
2725                         event < RTE_ETH_EVENT_MAX; event++) {
2726                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2727                                 event,
2728                                 eth_event_callback,
2729                                 NULL);
2730                 if (ret != 0) {
2731                         TESTPMD_LOG(ERR, "Failed to register callback for "
2732                                         "%s event\n", eth_event_desc[event]);
2733                         return -1;
2734                 }
2735         }
2736
2737         return 0;
2738 }
2739
2740 /* This function is used by the interrupt thread */
2741 static void
2742 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2743                              __rte_unused void *arg)
2744 {
2745         uint16_t port_id;
2746         int ret;
2747
2748         if (type >= RTE_DEV_EVENT_MAX) {
2749                 fprintf(stderr, "%s called upon invalid event %d\n",
2750                         __func__, type);
2751                 fflush(stderr);
2752         }
2753
2754         switch (type) {
2755         case RTE_DEV_EVENT_REMOVE:
2756                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2757                         device_name);
2758                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2759                 if (ret) {
2760                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2761                                 device_name);
2762                         return;
2763                 }
2764                 /*
2765                  * Because the user's callback is invoked in eal interrupt
2766                  * callback, the interrupt callback need to be finished before
2767                  * it can be unregistered when detaching device. So finish
2768                  * callback soon and use a deferred removal to detach device
2769                  * is need. It is a workaround, once the device detaching be
2770                  * moved into the eal in the future, the deferred removal could
2771                  * be deleted.
2772                  */
2773                 if (rte_eal_alarm_set(100000,
2774                                 rmv_port_callback, (void *)(intptr_t)port_id))
2775                         RTE_LOG(ERR, EAL,
2776                                 "Could not set up deferred device removal\n");
2777                 break;
2778         case RTE_DEV_EVENT_ADD:
2779                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2780                         device_name);
2781                 /* TODO: After finish kernel driver binding,
2782                  * begin to attach port.
2783                  */
2784                 break;
2785         default:
2786                 break;
2787         }
2788 }
2789
2790 static int
2791 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2792 {
2793         uint16_t i;
2794         int diag;
2795         uint8_t mapping_found = 0;
2796
2797         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2798                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2799                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2800                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2801                                         tx_queue_stats_mappings[i].queue_id,
2802                                         tx_queue_stats_mappings[i].stats_counter_id);
2803                         if (diag != 0)
2804                                 return diag;
2805                         mapping_found = 1;
2806                 }
2807         }
2808         if (mapping_found)
2809                 port->tx_queue_stats_mapping_enabled = 1;
2810         return 0;
2811 }
2812
2813 static int
2814 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2815 {
2816         uint16_t i;
2817         int diag;
2818         uint8_t mapping_found = 0;
2819
2820         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2821                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2822                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2823                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2824                                         rx_queue_stats_mappings[i].queue_id,
2825                                         rx_queue_stats_mappings[i].stats_counter_id);
2826                         if (diag != 0)
2827                                 return diag;
2828                         mapping_found = 1;
2829                 }
2830         }
2831         if (mapping_found)
2832                 port->rx_queue_stats_mapping_enabled = 1;
2833         return 0;
2834 }
2835
2836 static void
2837 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2838 {
2839         int diag = 0;
2840
2841         diag = set_tx_queue_stats_mapping_registers(pi, port);
2842         if (diag != 0) {
2843                 if (diag == -ENOTSUP) {
2844                         port->tx_queue_stats_mapping_enabled = 0;
2845                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2846                 }
2847                 else
2848                         rte_exit(EXIT_FAILURE,
2849                                         "set_tx_queue_stats_mapping_registers "
2850                                         "failed for port id=%d diag=%d\n",
2851                                         pi, diag);
2852         }
2853
2854         diag = set_rx_queue_stats_mapping_registers(pi, port);
2855         if (diag != 0) {
2856                 if (diag == -ENOTSUP) {
2857                         port->rx_queue_stats_mapping_enabled = 0;
2858                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2859                 }
2860                 else
2861                         rte_exit(EXIT_FAILURE,
2862                                         "set_rx_queue_stats_mapping_registers "
2863                                         "failed for port id=%d diag=%d\n",
2864                                         pi, diag);
2865         }
2866 }
2867
2868 static void
2869 rxtx_port_config(struct rte_port *port)
2870 {
2871         uint16_t qid;
2872         uint64_t offloads;
2873
2874         for (qid = 0; qid < nb_rxq; qid++) {
2875                 offloads = port->rx_conf[qid].offloads;
2876                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2877                 if (offloads != 0)
2878                         port->rx_conf[qid].offloads = offloads;
2879
2880                 /* Check if any Rx parameters have been passed */
2881                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2882                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2883
2884                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2885                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2886
2887                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2888                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2889
2890                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2891                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2892
2893                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2894                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2895
2896                 port->nb_rx_desc[qid] = nb_rxd;
2897         }
2898
2899         for (qid = 0; qid < nb_txq; qid++) {
2900                 offloads = port->tx_conf[qid].offloads;
2901                 port->tx_conf[qid] = port->dev_info.default_txconf;
2902                 if (offloads != 0)
2903                         port->tx_conf[qid].offloads = offloads;
2904
2905                 /* Check if any Tx parameters have been passed */
2906                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2907                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2908
2909                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2910                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2911
2912                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2913                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2914
2915                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2916                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2917
2918                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2919                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2920
2921                 port->nb_tx_desc[qid] = nb_txd;
2922         }
2923 }
2924
2925 void
2926 init_port_config(void)
2927 {
2928         portid_t pid;
2929         struct rte_port *port;
2930
2931         RTE_ETH_FOREACH_DEV(pid) {
2932                 port = &ports[pid];
2933                 port->dev_conf.fdir_conf = fdir_conf;
2934                 rte_eth_dev_info_get(pid, &port->dev_info);
2935                 if (nb_rxq > 1) {
2936                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2937                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2938                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2939                 } else {
2940                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2941                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2942                 }
2943
2944                 if (port->dcb_flag == 0) {
2945                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2946                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2947                         else
2948                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2949                 }
2950
2951                 rxtx_port_config(port);
2952
2953                 rte_eth_macaddr_get(pid, &port->eth_addr);
2954
2955                 map_port_queue_stats_mapping_registers(pid, port);
2956 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2957                 rte_pmd_ixgbe_bypass_init(pid);
2958 #endif
2959
2960                 if (lsc_interrupt &&
2961                     (rte_eth_devices[pid].data->dev_flags &
2962                      RTE_ETH_DEV_INTR_LSC))
2963                         port->dev_conf.intr_conf.lsc = 1;
2964                 if (rmv_interrupt &&
2965                     (rte_eth_devices[pid].data->dev_flags &
2966                      RTE_ETH_DEV_INTR_RMV))
2967                         port->dev_conf.intr_conf.rmv = 1;
2968         }
2969 }
2970
2971 void set_port_slave_flag(portid_t slave_pid)
2972 {
2973         struct rte_port *port;
2974
2975         port = &ports[slave_pid];
2976         port->slave_flag = 1;
2977 }
2978
2979 void clear_port_slave_flag(portid_t slave_pid)
2980 {
2981         struct rte_port *port;
2982
2983         port = &ports[slave_pid];
2984         port->slave_flag = 0;
2985 }
2986
2987 uint8_t port_is_bonding_slave(portid_t slave_pid)
2988 {
2989         struct rte_port *port;
2990
2991         port = &ports[slave_pid];
2992         if ((rte_eth_devices[slave_pid].data->dev_flags &
2993             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
2994                 return 1;
2995         return 0;
2996 }
2997
2998 const uint16_t vlan_tags[] = {
2999                 0,  1,  2,  3,  4,  5,  6,  7,
3000                 8,  9, 10, 11,  12, 13, 14, 15,
3001                 16, 17, 18, 19, 20, 21, 22, 23,
3002                 24, 25, 26, 27, 28, 29, 30, 31
3003 };
3004
3005 static  int
3006 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3007                  enum dcb_mode_enable dcb_mode,
3008                  enum rte_eth_nb_tcs num_tcs,
3009                  uint8_t pfc_en)
3010 {
3011         uint8_t i;
3012         int32_t rc;
3013         struct rte_eth_rss_conf rss_conf;
3014
3015         /*
3016          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3017          * given above, and the number of traffic classes available for use.
3018          */
3019         if (dcb_mode == DCB_VT_ENABLED) {
3020                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3021                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3022                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3023                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3024
3025                 /* VMDQ+DCB RX and TX configurations */
3026                 vmdq_rx_conf->enable_default_pool = 0;
3027                 vmdq_rx_conf->default_pool = 0;
3028                 vmdq_rx_conf->nb_queue_pools =
3029                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3030                 vmdq_tx_conf->nb_queue_pools =
3031                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3032
3033                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3034                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3035                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3036                         vmdq_rx_conf->pool_map[i].pools =
3037                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3038                 }
3039                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3040                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3041                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3042                 }
3043
3044                 /* set DCB mode of RX and TX of multiple queues */
3045                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
3046                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3047         } else {
3048                 struct rte_eth_dcb_rx_conf *rx_conf =
3049                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3050                 struct rte_eth_dcb_tx_conf *tx_conf =
3051                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3052
3053                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3054                 if (rc != 0)
3055                         return rc;
3056
3057                 rx_conf->nb_tcs = num_tcs;
3058                 tx_conf->nb_tcs = num_tcs;
3059
3060                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3061                         rx_conf->dcb_tc[i] = i % num_tcs;
3062                         tx_conf->dcb_tc[i] = i % num_tcs;
3063                 }
3064
3065                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
3066                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3067                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3068         }
3069
3070         if (pfc_en)
3071                 eth_conf->dcb_capability_en =
3072                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3073         else
3074                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3075
3076         return 0;
3077 }
3078
3079 int
3080 init_port_dcb_config(portid_t pid,
3081                      enum dcb_mode_enable dcb_mode,
3082                      enum rte_eth_nb_tcs num_tcs,
3083                      uint8_t pfc_en)
3084 {
3085         struct rte_eth_conf port_conf;
3086         struct rte_port *rte_port;
3087         int retval;
3088         uint16_t i;
3089
3090         rte_port = &ports[pid];
3091
3092         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3093         /* Enter DCB configuration status */
3094         dcb_config = 1;
3095
3096         port_conf.rxmode = rte_port->dev_conf.rxmode;
3097         port_conf.txmode = rte_port->dev_conf.txmode;
3098
3099         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3100         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3101         if (retval < 0)
3102                 return retval;
3103         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3104
3105         /* re-configure the device . */
3106         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3107         if (retval < 0)
3108                 return retval;
3109         rte_eth_dev_info_get(pid, &rte_port->dev_info);
3110
3111         /* If dev_info.vmdq_pool_base is greater than 0,
3112          * the queue id of vmdq pools is started after pf queues.
3113          */
3114         if (dcb_mode == DCB_VT_ENABLED &&
3115             rte_port->dev_info.vmdq_pool_base > 0) {
3116                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3117                         " for port %d.", pid);
3118                 return -1;
3119         }
3120
3121         /* Assume the ports in testpmd have the same dcb capability
3122          * and has the same number of rxq and txq in dcb mode
3123          */
3124         if (dcb_mode == DCB_VT_ENABLED) {
3125                 if (rte_port->dev_info.max_vfs > 0) {
3126                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3127                         nb_txq = rte_port->dev_info.nb_tx_queues;
3128                 } else {
3129                         nb_rxq = rte_port->dev_info.max_rx_queues;
3130                         nb_txq = rte_port->dev_info.max_tx_queues;
3131                 }
3132         } else {
3133                 /*if vt is disabled, use all pf queues */
3134                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3135                         nb_rxq = rte_port->dev_info.max_rx_queues;
3136                         nb_txq = rte_port->dev_info.max_tx_queues;
3137                 } else {
3138                         nb_rxq = (queueid_t)num_tcs;
3139                         nb_txq = (queueid_t)num_tcs;
3140
3141                 }
3142         }
3143         rx_free_thresh = 64;
3144
3145         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3146
3147         rxtx_port_config(rte_port);
3148         /* VLAN filter */
3149         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3150         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3151                 rx_vft_set(pid, vlan_tags[i], 1);
3152
3153         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
3154         map_port_queue_stats_mapping_registers(pid, rte_port);
3155
3156         rte_port->dcb_flag = 1;
3157
3158         return 0;
3159 }
3160
3161 static void
3162 init_port(void)
3163 {
3164         /* Configuration of Ethernet ports. */
3165         ports = rte_zmalloc("testpmd: ports",
3166                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3167                             RTE_CACHE_LINE_SIZE);
3168         if (ports == NULL) {
3169                 rte_exit(EXIT_FAILURE,
3170                                 "rte_zmalloc(%d struct rte_port) failed\n",
3171                                 RTE_MAX_ETHPORTS);
3172         }
3173
3174         /* Initialize ports NUMA structures */
3175         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3176         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3177         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3178 }
3179
3180 static void
3181 force_quit(void)
3182 {
3183         pmd_test_exit();
3184         prompt_exit();
3185 }
3186
3187 static void
3188 print_stats(void)
3189 {
3190         uint8_t i;
3191         const char clr[] = { 27, '[', '2', 'J', '\0' };
3192         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3193
3194         /* Clear screen and move to top left */
3195         printf("%s%s", clr, top_left);
3196
3197         printf("\nPort statistics ====================================");
3198         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3199                 nic_stats_display(fwd_ports_ids[i]);
3200
3201         fflush(stdout);
3202 }
3203
3204 static void
3205 signal_handler(int signum)
3206 {
3207         if (signum == SIGINT || signum == SIGTERM) {
3208                 printf("\nSignal %d received, preparing to exit...\n",
3209                                 signum);
3210 #ifdef RTE_LIBRTE_PDUMP
3211                 /* uninitialize packet capture framework */
3212                 rte_pdump_uninit();
3213 #endif
3214 #ifdef RTE_LIBRTE_LATENCY_STATS
3215                 rte_latencystats_uninit();
3216 #endif
3217                 force_quit();
3218                 /* Set flag to indicate the force termination. */
3219                 f_quit = 1;
3220                 /* exit with the expected status */
3221                 signal(signum, SIG_DFL);
3222                 kill(getpid(), signum);
3223         }
3224 }
3225
3226 int
3227 main(int argc, char** argv)
3228 {
3229         int diag;
3230         portid_t port_id;
3231         uint16_t count;
3232         int ret;
3233
3234         signal(SIGINT, signal_handler);
3235         signal(SIGTERM, signal_handler);
3236
3237         testpmd_logtype = rte_log_register("testpmd");
3238         if (testpmd_logtype < 0)
3239                 rte_panic("Cannot register log type");
3240         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3241
3242         diag = rte_eal_init(argc, argv);
3243         if (diag < 0)
3244                 rte_panic("Cannot init EAL\n");
3245
3246         ret = register_eth_event_callback();
3247         if (ret != 0)
3248                 rte_panic("Cannot register for ethdev events");
3249
3250 #ifdef RTE_LIBRTE_PDUMP
3251         /* initialize packet capture framework */
3252         rte_pdump_init();
3253 #endif
3254
3255         count = 0;
3256         RTE_ETH_FOREACH_DEV(port_id) {
3257                 ports_ids[count] = port_id;
3258                 count++;
3259         }
3260         nb_ports = (portid_t) count;
3261         if (nb_ports == 0)
3262                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3263
3264         /* allocate port structures, and init them */
3265         init_port();
3266
3267         set_def_fwd_config();
3268         if (nb_lcores == 0)
3269                 rte_panic("Empty set of forwarding logical cores - check the "
3270                           "core mask supplied in the command parameters\n");
3271
3272         /* Bitrate/latency stats disabled by default */
3273 #ifdef RTE_LIBRTE_BITRATE
3274         bitrate_enabled = 0;
3275 #endif
3276 #ifdef RTE_LIBRTE_LATENCY_STATS
3277         latencystats_enabled = 0;
3278 #endif
3279
3280         /* on FreeBSD, mlockall() is disabled by default */
3281 #ifdef RTE_EXEC_ENV_FREEBSD
3282         do_mlockall = 0;
3283 #else
3284         do_mlockall = 1;
3285 #endif
3286
3287         argc -= diag;
3288         argv += diag;
3289         if (argc > 1)
3290                 launch_args_parse(argc, argv);
3291
3292         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3293                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3294                         strerror(errno));
3295         }
3296
3297         if (tx_first && interactive)
3298                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3299                                 "interactive mode.\n");
3300
3301         if (tx_first && lsc_interrupt) {
3302                 printf("Warning: lsc_interrupt needs to be off when "
3303                                 " using tx_first. Disabling.\n");
3304                 lsc_interrupt = 0;
3305         }
3306
3307         if (!nb_rxq && !nb_txq)
3308                 printf("Warning: Either rx or tx queues should be non-zero\n");
3309
3310         if (nb_rxq > 1 && nb_rxq > nb_txq)
3311                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3312                        "but nb_txq=%d will prevent to fully test it.\n",
3313                        nb_rxq, nb_txq);
3314
3315         init_config();
3316
3317         if (hot_plug) {
3318                 ret = rte_dev_hotplug_handle_enable();
3319                 if (ret) {
3320                         RTE_LOG(ERR, EAL,
3321                                 "fail to enable hotplug handling.");
3322                         return -1;
3323                 }
3324
3325                 ret = rte_dev_event_monitor_start();
3326                 if (ret) {
3327                         RTE_LOG(ERR, EAL,
3328                                 "fail to start device event monitoring.");
3329                         return -1;
3330                 }
3331
3332                 ret = rte_dev_event_callback_register(NULL,
3333                         dev_event_callback, NULL);
3334                 if (ret) {
3335                         RTE_LOG(ERR, EAL,
3336                                 "fail  to register device event callback\n");
3337                         return -1;
3338                 }
3339         }
3340
3341         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3342                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3343
3344         /* set all ports to promiscuous mode by default */
3345         RTE_ETH_FOREACH_DEV(port_id)
3346                 rte_eth_promiscuous_enable(port_id);
3347
3348         /* Init metrics library */
3349         rte_metrics_init(rte_socket_id());
3350
3351 #ifdef RTE_LIBRTE_LATENCY_STATS
3352         if (latencystats_enabled != 0) {
3353                 int ret = rte_latencystats_init(1, NULL);
3354                 if (ret)
3355                         printf("Warning: latencystats init()"
3356                                 " returned error %d\n", ret);
3357                 printf("Latencystats running on lcore %d\n",
3358                         latencystats_lcore_id);
3359         }
3360 #endif
3361
3362         /* Setup bitrate stats */
3363 #ifdef RTE_LIBRTE_BITRATE
3364         if (bitrate_enabled != 0) {
3365                 bitrate_data = rte_stats_bitrate_create();
3366                 if (bitrate_data == NULL)
3367                         rte_exit(EXIT_FAILURE,
3368                                 "Could not allocate bitrate data.\n");
3369                 rte_stats_bitrate_reg(bitrate_data);
3370         }
3371 #endif
3372
3373 #ifdef RTE_LIBRTE_CMDLINE
3374         if (strlen(cmdline_filename) != 0)
3375                 cmdline_read_from_file(cmdline_filename);
3376
3377         if (interactive == 1) {
3378                 if (auto_start) {
3379                         printf("Start automatic packet forwarding\n");
3380                         start_packet_forwarding(0);
3381                 }
3382                 prompt();
3383                 pmd_test_exit();
3384         } else
3385 #endif
3386         {
3387                 char c;
3388                 int rc;
3389
3390                 f_quit = 0;
3391
3392                 printf("No commandline core given, start packet forwarding\n");
3393                 start_packet_forwarding(tx_first);
3394                 if (stats_period != 0) {
3395                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3396                         uint64_t timer_period;
3397
3398                         /* Convert to number of cycles */
3399                         timer_period = stats_period * rte_get_timer_hz();
3400
3401                         while (f_quit == 0) {
3402                                 cur_time = rte_get_timer_cycles();
3403                                 diff_time += cur_time - prev_time;
3404
3405                                 if (diff_time >= timer_period) {
3406                                         print_stats();
3407                                         /* Reset the timer */
3408                                         diff_time = 0;
3409                                 }
3410                                 /* Sleep to avoid unnecessary checks */
3411                                 prev_time = cur_time;
3412                                 sleep(1);
3413                         }
3414                 }
3415
3416                 printf("Press enter to exit\n");
3417                 rc = read(0, &c, 1);
3418                 pmd_test_exit();
3419                 if (rc < 0)
3420                         return 1;
3421         }
3422
3423         return 0;
3424 }