app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_malloc_heap.h>
  31 #include <rte_memory.h>
  32 #include <rte_memcpy.h>
  33 #include <rte_launch.h>
  34 #include <rte_eal.h>
  35 #include <rte_alarm.h>
  36 #include <rte_per_lcore.h>
  37 #include <rte_lcore.h>
  38 #include <rte_atomic.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_LIBRTE_IXGBE_PMD
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIBRTE_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #include <rte_metrics.h>
  58 #ifdef RTE_LIBRTE_BITRATE
  59 #include <rte_bitrate.h>
  60 #endif
  61 #ifdef RTE_LIBRTE_LATENCY_STATS
  62 #include <rte_latencystats.h>
  63 #endif
  64
  65 #include "testpmd.h"
  66
  67 #ifndef MAP_HUGETLB
  68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  69 #define HUGE_FLAG (0x40000)
  70 #else
  71 #define HUGE_FLAG MAP_HUGETLB
  72 #endif
  73
  74 #ifndef MAP_HUGE_SHIFT
  75 /* older kernels (or FreeBSD) will not have this define */
  76 #define HUGE_SHIFT (26)
  77 #else
  78 #define HUGE_SHIFT MAP_HUGE_SHIFT
  79 #endif
  80
  81 #define EXTMEM_HEAP_NAME "extmem"
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192 uint16_t mempool_flags;
 193
 194 struct fwd_config cur_fwd_config;
 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 196 uint32_t retry_enabled;
 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 199
 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 201 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 202                                       * specified on command-line. */
 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 204
 205 /*
 206  * In container, it cannot terminate the process which running with 'stats-period'
 207  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 208  */
 209 uint8_t f_quit;
 210
 211 /*
 212  * Configuration of packet segments used by the "txonly" processing engine.
 213  */
 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 216         TXONLY_DEF_PACKET_LEN,
 217 };
 218 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 219
 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 221 /**< Split policy for packets to TX. */
 222
 223 uint8_t txonly_multi_flow;
 224 /**< Whether multiple flows are generated in TXONLY mode. */
 225
 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 228
 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 230 uint8_t dcb_config = 0;
 231
 232 /* Whether the dcb is in testing status */
 233 uint8_t dcb_test = 0;
 234
 235 /*
 236  * Configurable number of RX/TX queues.
 237  */
 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 240
 241 /*
 242  * Configurable number of RX/TX ring descriptors.
 243  * Defaults are supplied by drivers via ethdev.
 244  */
 245 #define RTE_TEST_RX_DESC_DEFAULT 0
 246 #define RTE_TEST_TX_DESC_DEFAULT 0
 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 249
 250 #define RTE_PMD_PARAM_UNSET -1
 251 /*
 252  * Configurable values of RX and TX ring threshold registers.
 253  */
 254
 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 258
 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 262
 263 /*
 264  * Configurable value of RX free threshold.
 265  */
 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 267
 268 /*
 269  * Configurable value of RX drop enable.
 270  */
 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 272
 273 /*
 274  * Configurable value of TX free threshold.
 275  */
 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of TX RS bit threshold.
 280  */
 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of buffered packets before sending.
 285  */
 286 uint16_t noisy_tx_sw_bufsz;
 287
 288 /*
 289  * Configurable value of packet buffer timeout.
 290  */
 291 uint16_t noisy_tx_sw_buf_flush_time;
 292
 293 /*
 294  * Configurable value for size of VNF internal memory area
 295  * used for simulating noisy neighbour behaviour
 296  */
 297 uint64_t noisy_lkup_mem_sz;
 298
 299 /*
 300  * Configurable value of number of random writes done in
 301  * VNF simulation memory area.
 302  */
 303 uint64_t noisy_lkup_num_writes;
 304
 305 /*
 306  * Configurable value of number of random reads done in
 307  * VNF simulation memory area.
 308  */
 309 uint64_t noisy_lkup_num_reads;
 310
 311 /*
 312  * Configurable value of number of random reads/writes done in
 313  * VNF simulation memory area.
 314  */
 315 uint64_t noisy_lkup_num_reads_writes;
 316
 317 /*
 318  * Receive Side Scaling (RSS) configuration.
 319  */
 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 321
 322 /*
 323  * Port topology configuration
 324  */
 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 326
 327 /*
 328  * Avoids to flush all the RX streams before starts forwarding.
 329  */
 330 uint8_t no_flush_rx = 0; /* flush by default */
 331
 332 /*
 333  * Flow API isolated mode.
 334  */
 335 uint8_t flow_isolate_all;
 336
 337 /*
 338  * Avoids to check link status when starting/stopping a port.
 339  */
 340 uint8_t no_link_check = 0; /* check by default */
 341
 342 /*
 343  * Don't automatically start all ports in interactive mode.
 344  */
 345 uint8_t no_device_start = 0;
 346
 347 /*
 348  * Enable link status change notification
 349  */
 350 uint8_t lsc_interrupt = 1; /* enabled by default */
 351
 352 /*
 353  * Enable device removal notification.
 354  */
 355 uint8_t rmv_interrupt = 1; /* enabled by default */
 356
 357 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 358
 359 /* After attach, port setup is called on event or by iterator */
 360 bool setup_on_probe_event = true;
 361
 362 /* Pretty printing of ethdev events */
 363 static const char * const eth_event_desc[] = {
 364         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 365         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 366         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 367         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 368         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 369         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 370         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 371         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 372         [RTE_ETH_EVENT_NEW] = "device probed",
 373         [RTE_ETH_EVENT_DESTROY] = "device released",
 374         [RTE_ETH_EVENT_MAX] = NULL,
 375 };
 376
 377 /*
 378  * Display or mask ether events
 379  * Default to all events except VF_MBOX
 380  */
 381 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 382                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 383                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 384                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 385                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 386                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 387                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 388 /*
 389  * Decide if all memory are locked for performance.
 390  */
 391 int do_mlockall = 0;
 392
 393 /*
 394  * NIC bypass mode configuration options.
 395  */
 396
 397 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 398 /* The NIC bypass watchdog timeout. */
 399 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 400 #endif
 401
 402
 403 #ifdef RTE_LIBRTE_LATENCY_STATS
 404
 405 /*
 406  * Set when latency stats is enabled in the commandline
 407  */
 408 uint8_t latencystats_enabled;
 409
 410 /*
 411  * Lcore ID to serive latency statistics.
 412  */
 413 lcoreid_t latencystats_lcore_id = -1;
 414
 415 #endif
 416
 417 /*
 418  * Ethernet device configuration.
 419  */
 420 struct rte_eth_rxmode rx_mode = {
 421         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 422                 /**< Default maximum frame length. */
 423 };
 424
 425 struct rte_eth_txmode tx_mode = {
 426         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 427 };
 428
 429 struct rte_fdir_conf fdir_conf = {
 430         .mode = RTE_FDIR_MODE_NONE,
 431         .pballoc = RTE_FDIR_PBALLOC_64K,
 432         .status = RTE_FDIR_REPORT_STATUS,
 433         .mask = {
 434                 .vlan_tci_mask = 0xFFEF,
 435                 .ipv4_mask     = {
 436                         .src_ip = 0xFFFFFFFF,
 437                         .dst_ip = 0xFFFFFFFF,
 438                 },
 439                 .ipv6_mask     = {
 440                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 441                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 442                 },
 443                 .src_port_mask = 0xFFFF,
 444                 .dst_port_mask = 0xFFFF,
 445                 .mac_addr_byte_mask = 0xFF,
 446                 .tunnel_type_mask = 1,
 447                 .tunnel_id_mask = 0xFFFFFFFF,
 448         },
 449         .drop_queue = 127,
 450 };
 451
 452 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 453
 454 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 455 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 456
 457 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 458 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 459
 460 uint16_t nb_tx_queue_stats_mappings = 0;
 461 uint16_t nb_rx_queue_stats_mappings = 0;
 462
 463 /*
 464  * Display zero values by default for xstats
 465  */
 466 uint8_t xstats_hide_zero;
 467
 468 unsigned int num_sockets = 0;
 469 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 470
 471 #ifdef RTE_LIBRTE_BITRATE
 472 /* Bitrate statistics */
 473 struct rte_stats_bitrates *bitrate_data;
 474 lcoreid_t bitrate_lcore_id;
 475 uint8_t bitrate_enabled;
 476 #endif
 477
 478 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 479 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 480
 481 struct vxlan_encap_conf vxlan_encap_conf = {
 482         .select_ipv4 = 1,
 483         .select_vlan = 0,
 484         .select_tos_ttl = 0,
 485         .vni = "\x00\x00\x00",
 486         .udp_src = 0,
 487         .udp_dst = RTE_BE16(4789),
 488         .ipv4_src = RTE_IPV4(127, 0, 0, 1),
 489         .ipv4_dst = RTE_IPV4(255, 255, 255, 255),
 490         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 491                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 492         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 493                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 494         .vlan_tci = 0,
 495         .ip_tos = 0,
 496         .ip_ttl = 255,
 497         .eth_src = "\x00\x00\x00\x00\x00\x00",
 498         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 499 };
 500
 501 struct nvgre_encap_conf nvgre_encap_conf = {
 502         .select_ipv4 = 1,
 503         .select_vlan = 0,
 504         .tni = "\x00\x00\x00",
 505         .ipv4_src = RTE_IPV4(127, 0, 0, 1),
 506         .ipv4_dst = RTE_IPV4(255, 255, 255, 255),
 507         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 508                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 509         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 510                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 511         .vlan_tci = 0,
 512         .eth_src = "\x00\x00\x00\x00\x00\x00",
 513         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 514 };
 515
 516 /* Forward function declarations */
 517 static void setup_attached_port(portid_t pi);
 518 static void map_port_queue_stats_mapping_registers(portid_t pi,
 519                                                    struct rte_port *port);
 520 static void check_all_ports_link_status(uint32_t port_mask);
 521 static int eth_event_callback(portid_t port_id,
 522                               enum rte_eth_event_type type,
 523                               void *param, void *ret_param);
 524 static void dev_event_callback(const char *device_name,
 525                                 enum rte_dev_event_type type,
 526                                 void *param);
 527
 528 /*
 529  * Check if all the ports are started.
 530  * If yes, return positive value. If not, return zero.
 531  */
 532 static int all_ports_started(void);
 533
 534 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 535 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 536
 537 /*
 538  * Helper function to check if socket is already discovered.
 539  * If yes, return positive value. If not, return zero.
 540  */
 541 int
 542 new_socket_id(unsigned int socket_id)
 543 {
 544         unsigned int i;
 545
 546         for (i = 0; i < num_sockets; i++) {
 547                 if (socket_ids[i] == socket_id)
 548                         return 0;
 549         }
 550         return 1;
 551 }
 552
 553 /*
 554  * Setup default configuration.
 555  */
 556 static void
 557 set_default_fwd_lcores_config(void)
 558 {
 559         unsigned int i;
 560         unsigned int nb_lc;
 561         unsigned int sock_num;
 562
 563         nb_lc = 0;
 564         for (i = 0; i < RTE_MAX_LCORE; i++) {
 565                 if (!rte_lcore_is_enabled(i))
 566                         continue;
 567                 sock_num = rte_lcore_to_socket_id(i);
 568                 if (new_socket_id(sock_num)) {
 569                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 570                                 rte_exit(EXIT_FAILURE,
 571                                          "Total sockets greater than %u\n",
 572                                          RTE_MAX_NUMA_NODES);
 573                         }
 574                         socket_ids[num_sockets++] = sock_num;
 575                 }
 576                 if (i == rte_get_master_lcore())
 577                         continue;
 578                 fwd_lcores_cpuids[nb_lc++] = i;
 579         }
 580         nb_lcores = (lcoreid_t) nb_lc;
 581         nb_cfg_lcores = nb_lcores;
 582         nb_fwd_lcores = 1;
 583 }
 584
 585 static void
 586 set_def_peer_eth_addrs(void)
 587 {
 588         portid_t i;
 589
 590         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 591                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 592                 peer_eth_addrs[i].addr_bytes[5] = i;
 593         }
 594 }
 595
 596 static void
 597 set_default_fwd_ports_config(void)
 598 {
 599         portid_t pt_id;
 600         int i = 0;
 601
 602         RTE_ETH_FOREACH_DEV(pt_id) {
 603                 fwd_ports_ids[i++] = pt_id;
 604
 605                 /* Update sockets info according to the attached device */
 606                 int socket_id = rte_eth_dev_socket_id(pt_id);
 607                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 608                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 609                                 rte_exit(EXIT_FAILURE,
 610                                          "Total sockets greater than %u\n",
 611                                          RTE_MAX_NUMA_NODES);
 612                         }
 613                         socket_ids[num_sockets++] = socket_id;
 614                 }
 615         }
 616
 617         nb_cfg_ports = nb_ports;
 618         nb_fwd_ports = nb_ports;
 619 }
 620
 621 void
 622 set_def_fwd_config(void)
 623 {
 624         set_default_fwd_lcores_config();
 625         set_def_peer_eth_addrs();
 626         set_default_fwd_ports_config();
 627 }
 628
 629 /* extremely pessimistic estimation of memory required to create a mempool */
 630 static int
 631 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 632 {
 633         unsigned int n_pages, mbuf_per_pg, leftover;
 634         uint64_t total_mem, mbuf_mem, obj_sz;
 635
 636         /* there is no good way to predict how much space the mempool will
 637          * occupy because it will allocate chunks on the fly, and some of those
 638          * will come from default DPDK memory while some will come from our
 639          * external memory, so just assume 128MB will be enough for everyone.
 640          */
 641         uint64_t hdr_mem = 128 << 20;
 642
 643         /* account for possible non-contiguousness */
 644         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 645         if (obj_sz > pgsz) {
 646                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 647                 return -1;
 648         }
 649
 650         mbuf_per_pg = pgsz / obj_sz;
 651         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 652         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 653
 654         mbuf_mem = n_pages * pgsz;
 655
 656         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 657
 658         if (total_mem > SIZE_MAX) {
 659                 TESTPMD_LOG(ERR, "Memory size too big\n");
 660                 return -1;
 661         }
 662         *out = (size_t)total_mem;
 663
 664         return 0;
 665 }
 666
 667 static int
 668 pagesz_flags(uint64_t page_sz)
 669 {
 670         /* as per mmap() manpage, all page sizes are log2 of page size
 671          * shifted by MAP_HUGE_SHIFT
 672          */
 673         int log2 = rte_log2_u64(page_sz);
 674
 675         return (log2 << HUGE_SHIFT);
 676 }
 677
 678 static void *
 679 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 680 {
 681         void *addr;
 682         int flags;
 683
 684         /* allocate anonymous hugepages */
 685         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 686         if (huge)
 687                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 688
 689         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 690         if (addr == MAP_FAILED)
 691                 return NULL;
 692
 693         return addr;
 694 }
 695
 696 struct extmem_param {
 697         void *addr;
 698         size_t len;
 699         size_t pgsz;
 700         rte_iova_t *iova_table;
 701         unsigned int iova_table_len;
 702 };
 703
 704 static int
 705 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 706                 bool huge)
 707 {
 708         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 709                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 710         unsigned int cur_page, n_pages, pgsz_idx;
 711         size_t mem_sz, cur_pgsz;
 712         rte_iova_t *iovas = NULL;
 713         void *addr;
 714         int ret;
 715
 716         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 717                 /* skip anything that is too big */
 718                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 719                         continue;
 720
 721                 cur_pgsz = pgsizes[pgsz_idx];
 722
 723                 /* if we were told not to allocate hugepages, override */
 724                 if (!huge)
 725                         cur_pgsz = sysconf(_SC_PAGESIZE);
 726
 727                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 728                 if (ret < 0) {
 729                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 730                         return -1;
 731                 }
 732
 733                 /* allocate our memory */
 734                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 735
 736                 /* if we couldn't allocate memory with a specified page size,
 737                  * that doesn't mean we can't do it with other page sizes, so
 738                  * try another one.
 739                  */
 740                 if (addr == NULL)
 741                         continue;
 742
 743                 /* store IOVA addresses for every page in this memory area */
 744                 n_pages = mem_sz / cur_pgsz;
 745
 746                 iovas = malloc(sizeof(*iovas) * n_pages);
 747
 748                 if (iovas == NULL) {
 749                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 750                         goto fail;
 751                 }
 752                 /* lock memory if it's not huge pages */
 753                 if (!huge)
 754                         mlock(addr, mem_sz);
 755
 756                 /* populate IOVA addresses */
 757                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 758                         rte_iova_t iova;
 759                         size_t offset;
 760                         void *cur;
 761
 762                         offset = cur_pgsz * cur_page;
 763                         cur = RTE_PTR_ADD(addr, offset);
 764
 765                         /* touch the page before getting its IOVA */
 766                         *(volatile char *)cur = 0;
 767
 768                         iova = rte_mem_virt2iova(cur);
 769
 770                         iovas[cur_page] = iova;
 771                 }
 772
 773                 break;
 774         }
 775         /* if we couldn't allocate anything */
 776         if (iovas == NULL)
 777                 return -1;
 778
 779         param->addr = addr;
 780         param->len = mem_sz;
 781         param->pgsz = cur_pgsz;
 782         param->iova_table = iovas;
 783         param->iova_table_len = n_pages;
 784
 785         return 0;
 786 fail:
 787         if (iovas)
 788                 free(iovas);
 789         if (addr)
 790                 munmap(addr, mem_sz);
 791
 792         return -1;
 793 }
 794
 795 static int
 796 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 797 {
 798         struct extmem_param param;
 799         int socket_id, ret;
 800
 801         memset(&param, 0, sizeof(param));
 802
 803         /* check if our heap exists */
 804         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 805         if (socket_id < 0) {
 806                 /* create our heap */
 807                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 808                 if (ret < 0) {
 809                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 810                         return -1;
 811                 }
 812         }
 813
 814         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 815         if (ret < 0) {
 816                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 817                 return -1;
 818         }
 819
 820         /* we now have a valid memory area, so add it to heap */
 821         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 822                         param.addr, param.len, param.iova_table,
 823                         param.iova_table_len, param.pgsz);
 824
 825         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 826
 827         /* not needed any more */
 828         free(param.iova_table);
 829
 830         if (ret < 0) {
 831                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 832                 munmap(param.addr, param.len);
 833                 return -1;
 834         }
 835
 836         /* success */
 837
 838         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 839                         param.len >> 20);
 840
 841         return 0;
 842 }
 843 static void
 844 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 845              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 846 {
 847         uint16_t pid = 0;
 848         int ret;
 849
 850         RTE_ETH_FOREACH_DEV(pid) {
 851                 struct rte_eth_dev *dev =
 852                         &rte_eth_devices[pid];
 853
 854                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 855                                         memhdr->len);
 856                 if (ret) {
 857                         TESTPMD_LOG(DEBUG,
 858                                     "unable to DMA unmap addr 0x%p "
 859                                     "for device %s\n",
 860                                     memhdr->addr, dev->data->name);
 861                 }
 862         }
 863         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 864         if (ret) {
 865                 TESTPMD_LOG(DEBUG,
 866                             "unable to un-register addr 0x%p\n", memhdr->addr);
 867         }
 868 }
 869
 870 static void
 871 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 872            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 873 {
 874         uint16_t pid = 0;
 875         size_t page_size = sysconf(_SC_PAGESIZE);
 876         int ret;
 877
 878         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 879                                   page_size);
 880         if (ret) {
 881                 TESTPMD_LOG(DEBUG,
 882                             "unable to register addr 0x%p\n", memhdr->addr);
 883                 return;
 884         }
 885         RTE_ETH_FOREACH_DEV(pid) {
 886                 struct rte_eth_dev *dev =
 887                         &rte_eth_devices[pid];
 888
 889                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 890                                       memhdr->len);
 891                 if (ret) {
 892                         TESTPMD_LOG(DEBUG,
 893                                     "unable to DMA map addr 0x%p "
 894                                     "for device %s\n",
 895                                     memhdr->addr, dev->data->name);
 896                 }
 897         }
 898 }
 899
 900 /*
 901  * Configuration initialisation done once at init time.
 902  */
 903 static struct rte_mempool *
 904 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 905                  unsigned int socket_id)
 906 {
 907         char pool_name[RTE_MEMPOOL_NAMESIZE];
 908         struct rte_mempool *rte_mp = NULL;
 909         uint32_t mb_size;
 910
 911         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 912         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 913
 914         TESTPMD_LOG(INFO,
 915                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 916                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 917
 918         switch (mp_alloc_type) {
 919         case MP_ALLOC_NATIVE:
 920                 {
 921                         /* wrapper to rte_mempool_create() */
 922                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 923                                         rte_mbuf_best_mempool_ops());
 924                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 925                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 926                         break;
 927                 }
 928         case MP_ALLOC_ANON:
 929                 {
 930                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 931                                 mb_size, (unsigned int) mb_mempool_cache,
 932                                 sizeof(struct rte_pktmbuf_pool_private),
 933                                 socket_id, mempool_flags);
 934                         if (rte_mp == NULL)
 935                                 goto err;
 936
 937                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 938                                 rte_mempool_free(rte_mp);
 939                                 rte_mp = NULL;
 940                                 goto err;
 941                         }
 942                         rte_pktmbuf_pool_init(rte_mp, NULL);
 943                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 944                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 945                         break;
 946                 }
 947         case MP_ALLOC_XMEM:
 948         case MP_ALLOC_XMEM_HUGE:
 949                 {
 950                         int heap_socket;
 951                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 952
 953                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 954                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 955
 956                         heap_socket =
 957                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 958                         if (heap_socket < 0)
 959                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 960
 961                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 962                                         rte_mbuf_best_mempool_ops());
 963                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 964                                         mb_mempool_cache, 0, mbuf_seg_size,
 965                                         heap_socket);
 966                         break;
 967                 }
 968         default:
 969                 {
 970                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 971                 }
 972         }
 973
 974 err:
 975         if (rte_mp == NULL) {
 976                 rte_exit(EXIT_FAILURE,
 977                         "Creation of mbuf pool for socket %u failed: %s\n",
 978                         socket_id, rte_strerror(rte_errno));
 979         } else if (verbose_level > 0) {
 980                 rte_mempool_dump(stdout, rte_mp);
 981         }
 982         return rte_mp;
 983 }
 984
 985 /*
 986  * Check given socket id is valid or not with NUMA mode,
 987  * if valid, return 0, else return -1
 988  */
 989 static int
 990 check_socket_id(const unsigned int socket_id)
 991 {
 992         static int warning_once = 0;
 993
 994         if (new_socket_id(socket_id)) {
 995                 if (!warning_once && numa_support)
 996                         printf("Warning: NUMA should be configured manually by"
 997                                " using --port-numa-config and"
 998                                " --ring-numa-config parameters along with"
 999                                " --numa.\n");
1000                 warning_once = 1;
1001                 return -1;
1002         }
1003         return 0;
1004 }
1005
1006 /*
1007  * Get the allowed maximum number of RX queues.
1008  * *pid return the port id which has minimal value of
1009  * max_rx_queues in all ports.
1010  */
1011 queueid_t
1012 get_allowed_max_nb_rxq(portid_t *pid)
1013 {
1014         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
1015         bool max_rxq_valid = false;
1016         portid_t pi;
1017         struct rte_eth_dev_info dev_info;
1018
1019         RTE_ETH_FOREACH_DEV(pi) {
1020                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1021                         continue;
1022
1023                 max_rxq_valid = true;
1024                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1025                         allowed_max_rxq = dev_info.max_rx_queues;
1026                         *pid = pi;
1027                 }
1028         }
1029         return max_rxq_valid ? allowed_max_rxq : 0;
1030 }
1031
1032 /*
1033  * Check input rxq is valid or not.
1034  * If input rxq is not greater than any of maximum number
1035  * of RX queues of all ports, it is valid.
1036  * if valid, return 0, else return -1
1037  */
1038 int
1039 check_nb_rxq(queueid_t rxq)
1040 {
1041         queueid_t allowed_max_rxq;
1042         portid_t pid = 0;
1043
1044         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1045         if (rxq > allowed_max_rxq) {
1046                 printf("Fail: input rxq (%u) can't be greater "
1047                        "than max_rx_queues (%u) of port %u\n",
1048                        rxq,
1049                        allowed_max_rxq,
1050                        pid);
1051                 return -1;
1052         }
1053         return 0;
1054 }
1055
1056 /*
1057  * Get the allowed maximum number of TX queues.
1058  * *pid return the port id which has minimal value of
1059  * max_tx_queues in all ports.
1060  */
1061 queueid_t
1062 get_allowed_max_nb_txq(portid_t *pid)
1063 {
1064         queueid_t allowed_max_txq = MAX_QUEUE_ID;
1065         bool max_txq_valid = false;
1066         portid_t pi;
1067         struct rte_eth_dev_info dev_info;
1068
1069         RTE_ETH_FOREACH_DEV(pi) {
1070                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1071                         continue;
1072
1073                 max_txq_valid = true;
1074                 if (dev_info.max_tx_queues < allowed_max_txq) {
1075                         allowed_max_txq = dev_info.max_tx_queues;
1076                         *pid = pi;
1077                 }
1078         }
1079         return max_txq_valid ? allowed_max_txq : 0;
1080 }
1081
1082 /*
1083  * Check input txq is valid or not.
1084  * If input txq is not greater than any of maximum number
1085  * of TX queues of all ports, it is valid.
1086  * if valid, return 0, else return -1
1087  */
1088 int
1089 check_nb_txq(queueid_t txq)
1090 {
1091         queueid_t allowed_max_txq;
1092         portid_t pid = 0;
1093
1094         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1095         if (txq > allowed_max_txq) {
1096                 printf("Fail: input txq (%u) can't be greater "
1097                        "than max_tx_queues (%u) of port %u\n",
1098                        txq,
1099                        allowed_max_txq,
1100                        pid);
1101                 return -1;
1102         }
1103         return 0;
1104 }
1105
1106 static void
1107 init_config(void)
1108 {
1109         portid_t pid;
1110         struct rte_port *port;
1111         struct rte_mempool *mbp;
1112         unsigned int nb_mbuf_per_pool;
1113         lcoreid_t  lc_id;
1114         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1115         struct rte_gro_param gro_param;
1116         uint32_t gso_types;
1117         uint16_t data_size;
1118         bool warning = 0;
1119         int k;
1120         int ret;
1121
1122         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1123
1124         /* Configuration of logical cores. */
1125         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1126                                 sizeof(struct fwd_lcore *) * nb_lcores,
1127                                 RTE_CACHE_LINE_SIZE);
1128         if (fwd_lcores == NULL) {
1129                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1130                                                         "failed\n", nb_lcores);
1131         }
1132         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1133                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1134                                                sizeof(struct fwd_lcore),
1135                                                RTE_CACHE_LINE_SIZE);
1136                 if (fwd_lcores[lc_id] == NULL) {
1137                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1138                                                                 "failed\n");
1139                 }
1140                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1141         }
1142
1143         RTE_ETH_FOREACH_DEV(pid) {
1144                 port = &ports[pid];
1145                 /* Apply default TxRx configuration for all ports */
1146                 port->dev_conf.txmode = tx_mode;
1147                 port->dev_conf.rxmode = rx_mode;
1148
1149                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1150                 if (ret != 0)
1151                         rte_exit(EXIT_FAILURE,
1152                                  "rte_eth_dev_info_get() failed\n");
1153
1154                 if (!(port->dev_info.tx_offload_capa &
1155                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1156                         port->dev_conf.txmode.offloads &=
1157                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1158                 if (!(port->dev_info.tx_offload_capa &
1159                         DEV_TX_OFFLOAD_MATCH_METADATA))
1160                         port->dev_conf.txmode.offloads &=
1161                                 ~DEV_TX_OFFLOAD_MATCH_METADATA;
1162                 if (numa_support) {
1163                         if (port_numa[pid] != NUMA_NO_CONFIG)
1164                                 port_per_socket[port_numa[pid]]++;
1165                         else {
1166                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1167
1168                                 /*
1169                                  * if socket_id is invalid,
1170                                  * set to the first available socket.
1171                                  */
1172                                 if (check_socket_id(socket_id) < 0)
1173                                         socket_id = socket_ids[0];
1174                                 port_per_socket[socket_id]++;
1175                         }
1176                 }
1177
1178                 /* Apply Rx offloads configuration */
1179                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1180                         port->rx_conf[k].offloads =
1181                                 port->dev_conf.rxmode.offloads;
1182                 /* Apply Tx offloads configuration */
1183                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1184                         port->tx_conf[k].offloads =
1185                                 port->dev_conf.txmode.offloads;
1186
1187                 /* set flag to initialize port/queue */
1188                 port->need_reconfig = 1;
1189                 port->need_reconfig_queues = 1;
1190                 port->tx_metadata = 0;
1191
1192                 /* Check for maximum number of segments per MTU. Accordingly
1193                  * update the mbuf data size.
1194                  */
1195                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1196                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1197                         data_size = rx_mode.max_rx_pkt_len /
1198                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1199
1200                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1201                                                         mbuf_data_size) {
1202                                 mbuf_data_size = data_size +
1203                                                  RTE_PKTMBUF_HEADROOM;
1204                                 warning = 1;
1205                         }
1206                 }
1207         }
1208
1209         if (warning)
1210                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1211                             mbuf_data_size);
1212
1213         /*
1214          * Create pools of mbuf.
1215          * If NUMA support is disabled, create a single pool of mbuf in
1216          * socket 0 memory by default.
1217          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1218          *
1219          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1220          * nb_txd can be configured at run time.
1221          */
1222         if (param_total_num_mbufs)
1223                 nb_mbuf_per_pool = param_total_num_mbufs;
1224         else {
1225                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1226                         (nb_lcores * mb_mempool_cache) +
1227                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1228                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1229         }
1230
1231         if (numa_support) {
1232                 uint8_t i;
1233
1234                 for (i = 0; i < num_sockets; i++)
1235                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1236                                                        nb_mbuf_per_pool,
1237                                                        socket_ids[i]);
1238         } else {
1239                 if (socket_num == UMA_NO_CONFIG)
1240                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1241                                                        nb_mbuf_per_pool, 0);
1242                 else
1243                         mempools[socket_num] = mbuf_pool_create
1244                                                         (mbuf_data_size,
1245                                                          nb_mbuf_per_pool,
1246                                                          socket_num);
1247         }
1248
1249         init_port_config();
1250
1251         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1252                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1253         /*
1254          * Records which Mbuf pool to use by each logical core, if needed.
1255          */
1256         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1257                 mbp = mbuf_pool_find(
1258                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1259
1260                 if (mbp == NULL)
1261                         mbp = mbuf_pool_find(0);
1262                 fwd_lcores[lc_id]->mbp = mbp;
1263                 /* initialize GSO context */
1264                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1265                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1266                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1267                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1268                         RTE_ETHER_CRC_LEN;
1269                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1270         }
1271
1272         /* Configuration of packet forwarding streams. */
1273         if (init_fwd_streams() < 0)
1274                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1275
1276         fwd_config_setup();
1277
1278         /* create a gro context for each lcore */
1279         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1280         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1281         gro_param.max_item_per_flow = MAX_PKT_BURST;
1282         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1283                 gro_param.socket_id = rte_lcore_to_socket_id(
1284                                 fwd_lcores_cpuids[lc_id]);
1285                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1286                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1287                         rte_exit(EXIT_FAILURE,
1288                                         "rte_gro_ctx_create() failed\n");
1289                 }
1290         }
1291
1292 #if defined RTE_LIBRTE_PMD_SOFTNIC
1293         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1294                 RTE_ETH_FOREACH_DEV(pid) {
1295                         port = &ports[pid];
1296                         const char *driver = port->dev_info.driver_name;
1297
1298                         if (strcmp(driver, "net_softnic") == 0)
1299                                 port->softport.fwd_lcore_arg = fwd_lcores;
1300                 }
1301         }
1302 #endif
1303
1304 }
1305
1306
1307 void
1308 reconfig(portid_t new_port_id, unsigned socket_id)
1309 {
1310         struct rte_port *port;
1311         int ret;
1312
1313         /* Reconfiguration of Ethernet ports. */
1314         port = &ports[new_port_id];
1315
1316         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1317         if (ret != 0)
1318                 return;
1319
1320         /* set flag to initialize port/queue */
1321         port->need_reconfig = 1;
1322         port->need_reconfig_queues = 1;
1323         port->socket_id = socket_id;
1324
1325         init_port_config();
1326 }
1327
1328
1329 int
1330 init_fwd_streams(void)
1331 {
1332         portid_t pid;
1333         struct rte_port *port;
1334         streamid_t sm_id, nb_fwd_streams_new;
1335         queueid_t q;
1336
1337         /* set socket id according to numa or not */
1338         RTE_ETH_FOREACH_DEV(pid) {
1339                 port = &ports[pid];
1340                 if (nb_rxq > port->dev_info.max_rx_queues) {
1341                         printf("Fail: nb_rxq(%d) is greater than "
1342                                 "max_rx_queues(%d)\n", nb_rxq,
1343                                 port->dev_info.max_rx_queues);
1344                         return -1;
1345                 }
1346                 if (nb_txq > port->dev_info.max_tx_queues) {
1347                         printf("Fail: nb_txq(%d) is greater than "
1348                                 "max_tx_queues(%d)\n", nb_txq,
1349                                 port->dev_info.max_tx_queues);
1350                         return -1;
1351                 }
1352                 if (numa_support) {
1353                         if (port_numa[pid] != NUMA_NO_CONFIG)
1354                                 port->socket_id = port_numa[pid];
1355                         else {
1356                                 port->socket_id = rte_eth_dev_socket_id(pid);
1357
1358                                 /*
1359                                  * if socket_id is invalid,
1360                                  * set to the first available socket.
1361                                  */
1362                                 if (check_socket_id(port->socket_id) < 0)
1363                                         port->socket_id = socket_ids[0];
1364                         }
1365                 }
1366                 else {
1367                         if (socket_num == UMA_NO_CONFIG)
1368                                 port->socket_id = 0;
1369                         else
1370                                 port->socket_id = socket_num;
1371                 }
1372         }
1373
1374         q = RTE_MAX(nb_rxq, nb_txq);
1375         if (q == 0) {
1376                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1377                 return -1;
1378         }
1379         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1380         if (nb_fwd_streams_new == nb_fwd_streams)
1381                 return 0;
1382         /* clear the old */
1383         if (fwd_streams != NULL) {
1384                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1385                         if (fwd_streams[sm_id] == NULL)
1386                                 continue;
1387                         rte_free(fwd_streams[sm_id]);
1388                         fwd_streams[sm_id] = NULL;
1389                 }
1390                 rte_free(fwd_streams);
1391                 fwd_streams = NULL;
1392         }
1393
1394         /* init new */
1395         nb_fwd_streams = nb_fwd_streams_new;
1396         if (nb_fwd_streams) {
1397                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1398                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1399                         RTE_CACHE_LINE_SIZE);
1400                 if (fwd_streams == NULL)
1401                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1402                                  " (struct fwd_stream *)) failed\n",
1403                                  nb_fwd_streams);
1404
1405                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1406                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1407                                 " struct fwd_stream", sizeof(struct fwd_stream),
1408                                 RTE_CACHE_LINE_SIZE);
1409                         if (fwd_streams[sm_id] == NULL)
1410                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1411                                          "(struct fwd_stream) failed\n");
1412                 }
1413         }
1414
1415         return 0;
1416 }
1417
1418 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1419 static void
1420 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1421 {
1422         unsigned int total_burst;
1423         unsigned int nb_burst;
1424         unsigned int burst_stats[3];
1425         uint16_t pktnb_stats[3];
1426         uint16_t nb_pkt;
1427         int burst_percent[3];
1428
1429         /*
1430          * First compute the total number of packet bursts and the
1431          * two highest numbers of bursts of the same number of packets.
1432          */
1433         total_burst = 0;
1434         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1435         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1436         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1437                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1438                 if (nb_burst == 0)
1439                         continue;
1440                 total_burst += nb_burst;
1441                 if (nb_burst > burst_stats[0]) {
1442                         burst_stats[1] = burst_stats[0];
1443                         pktnb_stats[1] = pktnb_stats[0];
1444                         burst_stats[0] = nb_burst;
1445                         pktnb_stats[0] = nb_pkt;
1446                 } else if (nb_burst > burst_stats[1]) {
1447                         burst_stats[1] = nb_burst;
1448                         pktnb_stats[1] = nb_pkt;
1449                 }
1450         }
1451         if (total_burst == 0)
1452                 return;
1453         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1454         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1455                burst_percent[0], (int) pktnb_stats[0]);
1456         if (burst_stats[0] == total_burst) {
1457                 printf("]\n");
1458                 return;
1459         }
1460         if (burst_stats[0] + burst_stats[1] == total_burst) {
1461                 printf(" + %d%% of %d pkts]\n",
1462                        100 - burst_percent[0], pktnb_stats[1]);
1463                 return;
1464         }
1465         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1466         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1467         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1468                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1469                 return;
1470         }
1471         printf(" + %d%% of %d pkts + %d%% of others]\n",
1472                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1473 }
1474 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1475
1476 static void
1477 fwd_stream_stats_display(streamid_t stream_id)
1478 {
1479         struct fwd_stream *fs;
1480         static const char *fwd_top_stats_border = "-------";
1481
1482         fs = fwd_streams[stream_id];
1483         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1484             (fs->fwd_dropped == 0))
1485                 return;
1486         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1487                "TX Port=%2d/Queue=%2d %s\n",
1488                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1489                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1490         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1491                " TX-dropped: %-14"PRIu64,
1492                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1493
1494         /* if checksum mode */
1495         if (cur_fwd_eng == &csum_fwd_engine) {
1496                 printf("  RX- bad IP checksum: %-14"PRIu64
1497                        "  Rx- bad L4 checksum: %-14"PRIu64
1498                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1499                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1500                         fs->rx_bad_outer_l4_csum);
1501         } else {
1502                 printf("\n");
1503         }
1504
1505 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1506         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1507         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1508 #endif
1509 }
1510
1511 void
1512 fwd_stats_display(void)
1513 {
1514         static const char *fwd_stats_border = "----------------------";
1515         static const char *acc_stats_border = "+++++++++++++++";
1516         struct {
1517                 struct fwd_stream *rx_stream;
1518                 struct fwd_stream *tx_stream;
1519                 uint64_t tx_dropped;
1520                 uint64_t rx_bad_ip_csum;
1521                 uint64_t rx_bad_l4_csum;
1522                 uint64_t rx_bad_outer_l4_csum;
1523         } ports_stats[RTE_MAX_ETHPORTS];
1524         uint64_t total_rx_dropped = 0;
1525         uint64_t total_tx_dropped = 0;
1526         uint64_t total_rx_nombuf = 0;
1527         struct rte_eth_stats stats;
1528 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1529         uint64_t fwd_cycles = 0;
1530 #endif
1531         uint64_t total_recv = 0;
1532         uint64_t total_xmit = 0;
1533         struct rte_port *port;
1534         streamid_t sm_id;
1535         portid_t pt_id;
1536         int i;
1537
1538         memset(ports_stats, 0, sizeof(ports_stats));
1539
1540         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1541                 struct fwd_stream *fs = fwd_streams[sm_id];
1542
1543                 if (cur_fwd_config.nb_fwd_streams >
1544                     cur_fwd_config.nb_fwd_ports) {
1545                         fwd_stream_stats_display(sm_id);
1546                 } else {
1547                         ports_stats[fs->tx_port].tx_stream = fs;
1548                         ports_stats[fs->rx_port].rx_stream = fs;
1549                 }
1550
1551                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1552
1553                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1554                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1555                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1556                                 fs->rx_bad_outer_l4_csum;
1557
1558 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1559                 fwd_cycles += fs->core_cycles;
1560 #endif
1561         }
1562         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1563                 uint8_t j;
1564
1565                 pt_id = fwd_ports_ids[i];
1566                 port = &ports[pt_id];
1567
1568                 rte_eth_stats_get(pt_id, &stats);
1569                 stats.ipackets -= port->stats.ipackets;
1570                 stats.opackets -= port->stats.opackets;
1571                 stats.ibytes -= port->stats.ibytes;
1572                 stats.obytes -= port->stats.obytes;
1573                 stats.imissed -= port->stats.imissed;
1574                 stats.oerrors -= port->stats.oerrors;
1575                 stats.rx_nombuf -= port->stats.rx_nombuf;
1576
1577                 total_recv += stats.ipackets;
1578                 total_xmit += stats.opackets;
1579                 total_rx_dropped += stats.imissed;
1580                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1581                 total_tx_dropped += stats.oerrors;
1582                 total_rx_nombuf  += stats.rx_nombuf;
1583
1584                 printf("\n  %s Forward statistics for port %-2d %s\n",
1585                        fwd_stats_border, pt_id, fwd_stats_border);
1586
1587                 if (!port->rx_queue_stats_mapping_enabled &&
1588                     !port->tx_queue_stats_mapping_enabled) {
1589                         printf("  RX-packets: %-14"PRIu64
1590                                " RX-dropped: %-14"PRIu64
1591                                "RX-total: %-"PRIu64"\n",
1592                                stats.ipackets, stats.imissed,
1593                                stats.ipackets + stats.imissed);
1594
1595                         if (cur_fwd_eng == &csum_fwd_engine)
1596                                 printf("  Bad-ipcsum: %-14"PRIu64
1597                                        " Bad-l4csum: %-14"PRIu64
1598                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1599                                        ports_stats[pt_id].rx_bad_ip_csum,
1600                                        ports_stats[pt_id].rx_bad_l4_csum,
1601                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1602                         if (stats.ierrors + stats.rx_nombuf > 0) {
1603                                 printf("  RX-error: %-"PRIu64"\n",
1604                                        stats.ierrors);
1605                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1606                                        stats.rx_nombuf);
1607                         }
1608
1609                         printf("  TX-packets: %-14"PRIu64
1610                                " TX-dropped: %-14"PRIu64
1611                                "TX-total: %-"PRIu64"\n",
1612                                stats.opackets, ports_stats[pt_id].tx_dropped,
1613                                stats.opackets + ports_stats[pt_id].tx_dropped);
1614                 } else {
1615                         printf("  RX-packets:             %14"PRIu64
1616                                "    RX-dropped:%14"PRIu64
1617                                "    RX-total:%14"PRIu64"\n",
1618                                stats.ipackets, stats.imissed,
1619                                stats.ipackets + stats.imissed);
1620
1621                         if (cur_fwd_eng == &csum_fwd_engine)
1622                                 printf("  Bad-ipcsum:%14"PRIu64
1623                                        "    Bad-l4csum:%14"PRIu64
1624                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1625                                        ports_stats[pt_id].rx_bad_ip_csum,
1626                                        ports_stats[pt_id].rx_bad_l4_csum,
1627                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1628                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1629                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1630                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1631                                        stats.rx_nombuf);
1632                         }
1633
1634                         printf("  TX-packets:             %14"PRIu64
1635                                "    TX-dropped:%14"PRIu64
1636                                "    TX-total:%14"PRIu64"\n",
1637                                stats.opackets, ports_stats[pt_id].tx_dropped,
1638                                stats.opackets + ports_stats[pt_id].tx_dropped);
1639                 }
1640
1641 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1642                 if (ports_stats[pt_id].rx_stream)
1643                         pkt_burst_stats_display("RX",
1644                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1645                 if (ports_stats[pt_id].tx_stream)
1646                         pkt_burst_stats_display("TX",
1647                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1648 #endif
1649
1650                 if (port->rx_queue_stats_mapping_enabled) {
1651                         printf("\n");
1652                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1653                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1654                                        "     RX-errors:%14"PRIu64
1655                                        "    RX-bytes:%14"PRIu64"\n",
1656                                        j, stats.q_ipackets[j],
1657                                        stats.q_errors[j], stats.q_ibytes[j]);
1658                         }
1659                         printf("\n");
1660                 }
1661                 if (port->tx_queue_stats_mapping_enabled) {
1662                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1663                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1664                                        "                                 TX-bytes:%14"
1665                                        PRIu64"\n",
1666                                        j, stats.q_opackets[j],
1667                                        stats.q_obytes[j]);
1668                         }
1669                 }
1670
1671                 printf("  %s--------------------------------%s\n",
1672                        fwd_stats_border, fwd_stats_border);
1673         }
1674
1675         printf("\n  %s Accumulated forward statistics for all ports"
1676                "%s\n",
1677                acc_stats_border, acc_stats_border);
1678         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1679                "%-"PRIu64"\n"
1680                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1681                "%-"PRIu64"\n",
1682                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1683                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1684         if (total_rx_nombuf > 0)
1685                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1686         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1687                "%s\n",
1688                acc_stats_border, acc_stats_border);
1689 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1690         if (total_recv > 0)
1691                 printf("\n  CPU cycles/packet=%u (total cycles="
1692                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1693                        (unsigned int)(fwd_cycles / total_recv),
1694                        fwd_cycles, total_recv);
1695 #endif
1696 }
1697
1698 void
1699 fwd_stats_reset(void)
1700 {
1701         streamid_t sm_id;
1702         portid_t pt_id;
1703         int i;
1704
1705         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1706                 pt_id = fwd_ports_ids[i];
1707                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1708         }
1709         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1710                 struct fwd_stream *fs = fwd_streams[sm_id];
1711
1712                 fs->rx_packets = 0;
1713                 fs->tx_packets = 0;
1714                 fs->fwd_dropped = 0;
1715                 fs->rx_bad_ip_csum = 0;
1716                 fs->rx_bad_l4_csum = 0;
1717                 fs->rx_bad_outer_l4_csum = 0;
1718
1719 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1720                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1721                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1722 #endif
1723 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1724                 fs->core_cycles = 0;
1725 #endif
1726         }
1727 }
1728
1729 static void
1730 flush_fwd_rx_queues(void)
1731 {
1732         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1733         portid_t  rxp;
1734         portid_t port_id;
1735         queueid_t rxq;
1736         uint16_t  nb_rx;
1737         uint16_t  i;
1738         uint8_t   j;
1739         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1740         uint64_t timer_period;
1741
1742         /* convert to number of cycles */
1743         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1744
1745         for (j = 0; j < 2; j++) {
1746                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1747                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1748                                 port_id = fwd_ports_ids[rxp];
1749                                 /**
1750                                 * testpmd can stuck in the below do while loop
1751                                 * if rte_eth_rx_burst() always returns nonzero
1752                                 * packets. So timer is added to exit this loop
1753                                 * after 1sec timer expiry.
1754                                 */
1755                                 prev_tsc = rte_rdtsc();
1756                                 do {
1757                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1758                                                 pkts_burst, MAX_PKT_BURST);
1759                                         for (i = 0; i < nb_rx; i++)
1760                                                 rte_pktmbuf_free(pkts_burst[i]);
1761
1762                                         cur_tsc = rte_rdtsc();
1763                                         diff_tsc = cur_tsc - prev_tsc;
1764                                         timer_tsc += diff_tsc;
1765                                 } while ((nb_rx > 0) &&
1766                                         (timer_tsc < timer_period));
1767                                 timer_tsc = 0;
1768                         }
1769                 }
1770                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1771         }
1772 }
1773
1774 static void
1775 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1776 {
1777         struct fwd_stream **fsm;
1778         streamid_t nb_fs;
1779         streamid_t sm_id;
1780 #ifdef RTE_LIBRTE_BITRATE
1781         uint64_t tics_per_1sec;
1782         uint64_t tics_datum;
1783         uint64_t tics_current;
1784         uint16_t i, cnt_ports;
1785
1786         cnt_ports = nb_ports;
1787         tics_datum = rte_rdtsc();
1788         tics_per_1sec = rte_get_timer_hz();
1789 #endif
1790         fsm = &fwd_streams[fc->stream_idx];
1791         nb_fs = fc->stream_nb;
1792         do {
1793                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1794                         (*pkt_fwd)(fsm[sm_id]);
1795 #ifdef RTE_LIBRTE_BITRATE
1796                 if (bitrate_enabled != 0 &&
1797                                 bitrate_lcore_id == rte_lcore_id()) {
1798                         tics_current = rte_rdtsc();
1799                         if (tics_current - tics_datum >= tics_per_1sec) {
1800                                 /* Periodic bitrate calculation */
1801                                 for (i = 0; i < cnt_ports; i++)
1802                                         rte_stats_bitrate_calc(bitrate_data,
1803                                                 ports_ids[i]);
1804                                 tics_datum = tics_current;
1805                         }
1806                 }
1807 #endif
1808 #ifdef RTE_LIBRTE_LATENCY_STATS
1809                 if (latencystats_enabled != 0 &&
1810                                 latencystats_lcore_id == rte_lcore_id())
1811                         rte_latencystats_update();
1812 #endif
1813
1814         } while (! fc->stopped);
1815 }
1816
1817 static int
1818 start_pkt_forward_on_core(void *fwd_arg)
1819 {
1820         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1821                              cur_fwd_config.fwd_eng->packet_fwd);
1822         return 0;
1823 }
1824
1825 /*
1826  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1827  * Used to start communication flows in network loopback test configurations.
1828  */
1829 static int
1830 run_one_txonly_burst_on_core(void *fwd_arg)
1831 {
1832         struct fwd_lcore *fwd_lc;
1833         struct fwd_lcore tmp_lcore;
1834
1835         fwd_lc = (struct fwd_lcore *) fwd_arg;
1836         tmp_lcore = *fwd_lc;
1837         tmp_lcore.stopped = 1;
1838         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1839         return 0;
1840 }
1841
1842 /*
1843  * Launch packet forwarding:
1844  *     - Setup per-port forwarding context.
1845  *     - launch logical cores with their forwarding configuration.
1846  */
1847 static void
1848 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1849 {
1850         port_fwd_begin_t port_fwd_begin;
1851         unsigned int i;
1852         unsigned int lc_id;
1853         int diag;
1854
1855         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1856         if (port_fwd_begin != NULL) {
1857                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1858                         (*port_fwd_begin)(fwd_ports_ids[i]);
1859         }
1860         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1861                 lc_id = fwd_lcores_cpuids[i];
1862                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1863                         fwd_lcores[i]->stopped = 0;
1864                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1865                                                      fwd_lcores[i], lc_id);
1866                         if (diag != 0)
1867                                 printf("launch lcore %u failed - diag=%d\n",
1868                                        lc_id, diag);
1869                 }
1870         }
1871 }
1872
1873 /*
1874  * Launch packet forwarding configuration.
1875  */
1876 void
1877 start_packet_forwarding(int with_tx_first)
1878 {
1879         port_fwd_begin_t port_fwd_begin;
1880         port_fwd_end_t  port_fwd_end;
1881         struct rte_port *port;
1882         unsigned int i;
1883         portid_t   pt_id;
1884
1885         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1886                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1887
1888         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1889                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1890
1891         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1892                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1893                 (!nb_rxq || !nb_txq))
1894                 rte_exit(EXIT_FAILURE,
1895                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1896                         cur_fwd_eng->fwd_mode_name);
1897
1898         if (all_ports_started() == 0) {
1899                 printf("Not all ports were started\n");
1900                 return;
1901         }
1902         if (test_done == 0) {
1903                 printf("Packet forwarding already started\n");
1904                 return;
1905         }
1906
1907
1908         if(dcb_test) {
1909                 for (i = 0; i < nb_fwd_ports; i++) {
1910                         pt_id = fwd_ports_ids[i];
1911                         port = &ports[pt_id];
1912                         if (!port->dcb_flag) {
1913                                 printf("In DCB mode, all forwarding ports must "
1914                                        "be configured in this mode.\n");
1915                                 return;
1916                         }
1917                 }
1918                 if (nb_fwd_lcores == 1) {
1919                         printf("In DCB mode,the nb forwarding cores "
1920                                "should be larger than 1.\n");
1921                         return;
1922                 }
1923         }
1924         test_done = 0;
1925
1926         fwd_config_setup();
1927
1928         if(!no_flush_rx)
1929                 flush_fwd_rx_queues();
1930
1931         pkt_fwd_config_display(&cur_fwd_config);
1932         rxtx_config_display();
1933
1934         fwd_stats_reset();
1935         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1936                 pt_id = fwd_ports_ids[i];
1937                 port = &ports[pt_id];
1938                 map_port_queue_stats_mapping_registers(pt_id, port);
1939         }
1940         if (with_tx_first) {
1941                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1942                 if (port_fwd_begin != NULL) {
1943                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1944                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1945                 }
1946                 while (with_tx_first--) {
1947                         launch_packet_forwarding(
1948                                         run_one_txonly_burst_on_core);
1949                         rte_eal_mp_wait_lcore();
1950                 }
1951                 port_fwd_end = tx_only_engine.port_fwd_end;
1952                 if (port_fwd_end != NULL) {
1953                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1954                                 (*port_fwd_end)(fwd_ports_ids[i]);
1955                 }
1956         }
1957         launch_packet_forwarding(start_pkt_forward_on_core);
1958 }
1959
1960 void
1961 stop_packet_forwarding(void)
1962 {
1963         port_fwd_end_t port_fwd_end;
1964         lcoreid_t lc_id;
1965         portid_t pt_id;
1966         int i;
1967
1968         if (test_done) {
1969                 printf("Packet forwarding not started\n");
1970                 return;
1971         }
1972         printf("Telling cores to stop...");
1973         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1974                 fwd_lcores[lc_id]->stopped = 1;
1975         printf("\nWaiting for lcores to finish...\n");
1976         rte_eal_mp_wait_lcore();
1977         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1978         if (port_fwd_end != NULL) {
1979                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1980                         pt_id = fwd_ports_ids[i];
1981                         (*port_fwd_end)(pt_id);
1982                 }
1983         }
1984
1985         fwd_stats_display();
1986
1987         printf("\nDone.\n");
1988         test_done = 1;
1989 }
1990
1991 void
1992 dev_set_link_up(portid_t pid)
1993 {
1994         if (rte_eth_dev_set_link_up(pid) < 0)
1995                 printf("\nSet link up fail.\n");
1996 }
1997
1998 void
1999 dev_set_link_down(portid_t pid)
2000 {
2001         if (rte_eth_dev_set_link_down(pid) < 0)
2002                 printf("\nSet link down fail.\n");
2003 }
2004
2005 static int
2006 all_ports_started(void)
2007 {
2008         portid_t pi;
2009         struct rte_port *port;
2010
2011         RTE_ETH_FOREACH_DEV(pi) {
2012                 port = &ports[pi];
2013                 /* Check if there is a port which is not started */
2014                 if ((port->port_status != RTE_PORT_STARTED) &&
2015                         (port->slave_flag == 0))
2016                         return 0;
2017         }
2018
2019         /* No port is not started */
2020         return 1;
2021 }
2022
2023 int
2024 port_is_stopped(portid_t port_id)
2025 {
2026         struct rte_port *port = &ports[port_id];
2027
2028         if ((port->port_status != RTE_PORT_STOPPED) &&
2029             (port->slave_flag == 0))
2030                 return 0;
2031         return 1;
2032 }
2033
2034 int
2035 all_ports_stopped(void)
2036 {
2037         portid_t pi;
2038
2039         RTE_ETH_FOREACH_DEV(pi) {
2040                 if (!port_is_stopped(pi))
2041                         return 0;
2042         }
2043
2044         return 1;
2045 }
2046
2047 int
2048 port_is_started(portid_t port_id)
2049 {
2050         if (port_id_is_invalid(port_id, ENABLED_WARN))
2051                 return 0;
2052
2053         if (ports[port_id].port_status != RTE_PORT_STARTED)
2054                 return 0;
2055
2056         return 1;
2057 }
2058
2059 int
2060 start_port(portid_t pid)
2061 {
2062         int diag, need_check_link_status = -1;
2063         portid_t pi;
2064         queueid_t qi;
2065         struct rte_port *port;
2066         struct rte_ether_addr mac_addr;
2067
2068         if (port_id_is_invalid(pid, ENABLED_WARN))
2069                 return 0;
2070
2071         if(dcb_config)
2072                 dcb_test = 1;
2073         RTE_ETH_FOREACH_DEV(pi) {
2074                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2075                         continue;
2076
2077                 need_check_link_status = 0;
2078                 port = &ports[pi];
2079                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2080                                                  RTE_PORT_HANDLING) == 0) {
2081                         printf("Port %d is now not stopped\n", pi);
2082                         continue;
2083                 }
2084
2085                 if (port->need_reconfig > 0) {
2086                         port->need_reconfig = 0;
2087
2088                         if (flow_isolate_all) {
2089                                 int ret = port_flow_isolate(pi, 1);
2090                                 if (ret) {
2091                                         printf("Failed to apply isolated"
2092                                                " mode on port %d\n", pi);
2093                                         return -1;
2094                                 }
2095                         }
2096                         configure_rxtx_dump_callbacks(0);
2097                         printf("Configuring Port %d (socket %u)\n", pi,
2098                                         port->socket_id);
2099                         /* configure port */
2100                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
2101                                                 &(port->dev_conf));
2102                         if (diag != 0) {
2103                                 if (rte_atomic16_cmpset(&(port->port_status),
2104                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2105                                         printf("Port %d can not be set back "
2106                                                         "to stopped\n", pi);
2107                                 printf("Fail to configure port %d\n", pi);
2108                                 /* try to reconfigure port next time */
2109                                 port->need_reconfig = 1;
2110                                 return -1;
2111                         }
2112                 }
2113                 if (port->need_reconfig_queues > 0) {
2114                         port->need_reconfig_queues = 0;
2115                         /* setup tx queues */
2116                         for (qi = 0; qi < nb_txq; qi++) {
2117                                 if ((numa_support) &&
2118                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2119                                         diag = rte_eth_tx_queue_setup(pi, qi,
2120                                                 port->nb_tx_desc[qi],
2121                                                 txring_numa[pi],
2122                                                 &(port->tx_conf[qi]));
2123                                 else
2124                                         diag = rte_eth_tx_queue_setup(pi, qi,
2125                                                 port->nb_tx_desc[qi],
2126                                                 port->socket_id,
2127                                                 &(port->tx_conf[qi]));
2128
2129                                 if (diag == 0)
2130                                         continue;
2131
2132                                 /* Fail to setup tx queue, return */
2133                                 if (rte_atomic16_cmpset(&(port->port_status),
2134                                                         RTE_PORT_HANDLING,
2135                                                         RTE_PORT_STOPPED) == 0)
2136                                         printf("Port %d can not be set back "
2137                                                         "to stopped\n", pi);
2138                                 printf("Fail to configure port %d tx queues\n",
2139                                        pi);
2140                                 /* try to reconfigure queues next time */
2141                                 port->need_reconfig_queues = 1;
2142                                 return -1;
2143                         }
2144                         for (qi = 0; qi < nb_rxq; qi++) {
2145                                 /* setup rx queues */
2146                                 if ((numa_support) &&
2147                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2148                                         struct rte_mempool * mp =
2149                                                 mbuf_pool_find(rxring_numa[pi]);
2150                                         if (mp == NULL) {
2151                                                 printf("Failed to setup RX queue:"
2152                                                         "No mempool allocation"
2153                                                         " on the socket %d\n",
2154                                                         rxring_numa[pi]);
2155                                                 return -1;
2156                                         }
2157
2158                                         diag = rte_eth_rx_queue_setup(pi, qi,
2159                                              port->nb_rx_desc[qi],
2160                                              rxring_numa[pi],
2161                                              &(port->rx_conf[qi]),
2162                                              mp);
2163                                 } else {
2164                                         struct rte_mempool *mp =
2165                                                 mbuf_pool_find(port->socket_id);
2166                                         if (mp == NULL) {
2167                                                 printf("Failed to setup RX queue:"
2168                                                         "No mempool allocation"
2169                                                         " on the socket %d\n",
2170                                                         port->socket_id);
2171                                                 return -1;
2172                                         }
2173                                         diag = rte_eth_rx_queue_setup(pi, qi,
2174                                              port->nb_rx_desc[qi],
2175                                              port->socket_id,
2176                                              &(port->rx_conf[qi]),
2177                                              mp);
2178                                 }
2179                                 if (diag == 0)
2180                                         continue;
2181
2182                                 /* Fail to setup rx queue, return */
2183                                 if (rte_atomic16_cmpset(&(port->port_status),
2184                                                         RTE_PORT_HANDLING,
2185                                                         RTE_PORT_STOPPED) == 0)
2186                                         printf("Port %d can not be set back "
2187                                                         "to stopped\n", pi);
2188                                 printf("Fail to configure port %d rx queues\n",
2189                                        pi);
2190                                 /* try to reconfigure queues next time */
2191                                 port->need_reconfig_queues = 1;
2192                                 return -1;
2193                         }
2194                 }
2195                 configure_rxtx_dump_callbacks(verbose_level);
2196                 /* start port */
2197                 if (rte_eth_dev_start(pi) < 0) {
2198                         printf("Fail to start port %d\n", pi);
2199
2200                         /* Fail to setup rx queue, return */
2201                         if (rte_atomic16_cmpset(&(port->port_status),
2202                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2203                                 printf("Port %d can not be set back to "
2204                                                         "stopped\n", pi);
2205                         continue;
2206                 }
2207
2208                 if (rte_atomic16_cmpset(&(port->port_status),
2209                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2210                         printf("Port %d can not be set into started\n", pi);
2211
2212                 rte_eth_macaddr_get(pi, &mac_addr);
2213                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2214                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2215                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2216                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2217
2218                 /* at least one port started, need checking link status */
2219                 need_check_link_status = 1;
2220         }
2221
2222         if (need_check_link_status == 1 && !no_link_check)
2223                 check_all_ports_link_status(RTE_PORT_ALL);
2224         else if (need_check_link_status == 0)
2225                 printf("Please stop the ports first\n");
2226
2227         printf("Done\n");
2228         return 0;
2229 }
2230
2231 void
2232 stop_port(portid_t pid)
2233 {
2234         portid_t pi;
2235         struct rte_port *port;
2236         int need_check_link_status = 0;
2237
2238         if (dcb_test) {
2239                 dcb_test = 0;
2240                 dcb_config = 0;
2241         }
2242
2243         if (port_id_is_invalid(pid, ENABLED_WARN))
2244                 return;
2245
2246         printf("Stopping ports...\n");
2247
2248         RTE_ETH_FOREACH_DEV(pi) {
2249                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2250                         continue;
2251
2252                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2253                         printf("Please remove port %d from forwarding configuration.\n", pi);
2254                         continue;
2255                 }
2256
2257                 if (port_is_bonding_slave(pi)) {
2258                         printf("Please remove port %d from bonded device.\n", pi);
2259                         continue;
2260                 }
2261
2262                 port = &ports[pi];
2263                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2264                                                 RTE_PORT_HANDLING) == 0)
2265                         continue;
2266
2267                 rte_eth_dev_stop(pi);
2268
2269                 if (rte_atomic16_cmpset(&(port->port_status),
2270                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2271                         printf("Port %d can not be set into stopped\n", pi);
2272                 need_check_link_status = 1;
2273         }
2274         if (need_check_link_status && !no_link_check)
2275                 check_all_ports_link_status(RTE_PORT_ALL);
2276
2277         printf("Done\n");
2278 }
2279
2280 static void
2281 remove_invalid_ports_in(portid_t *array, portid_t *total)
2282 {
2283         portid_t i;
2284         portid_t new_total = 0;
2285
2286         for (i = 0; i < *total; i++)
2287                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2288                         array[new_total] = array[i];
2289                         new_total++;
2290                 }
2291         *total = new_total;
2292 }
2293
2294 static void
2295 remove_invalid_ports(void)
2296 {
2297         remove_invalid_ports_in(ports_ids, &nb_ports);
2298         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2299         nb_cfg_ports = nb_fwd_ports;
2300 }
2301
2302 void
2303 close_port(portid_t pid)
2304 {
2305         portid_t pi;
2306         struct rte_port *port;
2307
2308         if (port_id_is_invalid(pid, ENABLED_WARN))
2309                 return;
2310
2311         printf("Closing ports...\n");
2312
2313         RTE_ETH_FOREACH_DEV(pi) {
2314                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2315                         continue;
2316
2317                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2318                         printf("Please remove port %d from forwarding configuration.\n", pi);
2319                         continue;
2320                 }
2321
2322                 if (port_is_bonding_slave(pi)) {
2323                         printf("Please remove port %d from bonded device.\n", pi);
2324                         continue;
2325                 }
2326
2327                 port = &ports[pi];
2328                 if (rte_atomic16_cmpset(&(port->port_status),
2329                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2330                         printf("Port %d is already closed\n", pi);
2331                         continue;
2332                 }
2333
2334                 if (rte_atomic16_cmpset(&(port->port_status),
2335                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2336                         printf("Port %d is now not stopped\n", pi);
2337                         continue;
2338                 }
2339
2340                 if (port->flow_list)
2341                         port_flow_flush(pi);
2342                 rte_eth_dev_close(pi);
2343
2344                 remove_invalid_ports();
2345
2346                 if (rte_atomic16_cmpset(&(port->port_status),
2347                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2348                         printf("Port %d cannot be set to closed\n", pi);
2349         }
2350
2351         printf("Done\n");
2352 }
2353
2354 void
2355 reset_port(portid_t pid)
2356 {
2357         int diag;
2358         portid_t pi;
2359         struct rte_port *port;
2360
2361         if (port_id_is_invalid(pid, ENABLED_WARN))
2362                 return;
2363
2364         printf("Resetting ports...\n");
2365
2366         RTE_ETH_FOREACH_DEV(pi) {
2367                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2368                         continue;
2369
2370                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2371                         printf("Please remove port %d from forwarding "
2372                                "configuration.\n", pi);
2373                         continue;
2374                 }
2375
2376                 if (port_is_bonding_slave(pi)) {
2377                         printf("Please remove port %d from bonded device.\n",
2378                                pi);
2379                         continue;
2380                 }
2381
2382                 diag = rte_eth_dev_reset(pi);
2383                 if (diag == 0) {
2384                         port = &ports[pi];
2385                         port->need_reconfig = 1;
2386                         port->need_reconfig_queues = 1;
2387                 } else {
2388                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2389                 }
2390         }
2391
2392         printf("Done\n");
2393 }
2394
2395 void
2396 attach_port(char *identifier)
2397 {
2398         portid_t pi;
2399         struct rte_dev_iterator iterator;
2400
2401         printf("Attaching a new port...\n");
2402
2403         if (identifier == NULL) {
2404                 printf("Invalid parameters are specified\n");
2405                 return;
2406         }
2407
2408         if (rte_dev_probe(identifier) < 0) {
2409                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2410                 return;
2411         }
2412
2413         /* first attach mode: event */
2414         if (setup_on_probe_event) {
2415                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2416                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2417                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2418                                         ports[pi].need_setup != 0)
2419                                 setup_attached_port(pi);
2420                 return;
2421         }
2422
2423         /* second attach mode: iterator */
2424         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2425                 /* setup ports matching the devargs used for probing */
2426                 if (port_is_forwarding(pi))
2427                         continue; /* port was already attached before */
2428                 setup_attached_port(pi);
2429         }
2430 }
2431
2432 static void
2433 setup_attached_port(portid_t pi)
2434 {
2435         unsigned int socket_id;
2436         int ret;
2437
2438         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2439         /* if socket_id is invalid, set to the first available socket. */
2440         if (check_socket_id(socket_id) < 0)
2441                 socket_id = socket_ids[0];
2442         reconfig(pi, socket_id);
2443         ret = rte_eth_promiscuous_enable(pi);
2444         if (ret != 0)
2445                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2446                         pi, rte_strerror(-ret));
2447
2448         ports_ids[nb_ports++] = pi;
2449         fwd_ports_ids[nb_fwd_ports++] = pi;
2450         nb_cfg_ports = nb_fwd_ports;
2451         ports[pi].need_setup = 0;
2452         ports[pi].port_status = RTE_PORT_STOPPED;
2453
2454         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2455         printf("Done\n");
2456 }
2457
2458 void
2459 detach_port_device(portid_t port_id)
2460 {
2461         struct rte_device *dev;
2462         portid_t sibling;
2463
2464         printf("Removing a device...\n");
2465
2466         dev = rte_eth_devices[port_id].device;
2467         if (dev == NULL) {
2468                 printf("Device already removed\n");
2469                 return;
2470         }
2471
2472         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2473                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2474                         printf("Port not stopped\n");
2475                         return;
2476                 }
2477                 printf("Port was not closed\n");
2478                 if (ports[port_id].flow_list)
2479                         port_flow_flush(port_id);
2480         }
2481
2482         if (rte_dev_remove(dev) < 0) {
2483                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2484                 return;
2485         }
2486         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2487                 /* reset mapping between old ports and removed device */
2488                 rte_eth_devices[sibling].device = NULL;
2489                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2490                         /* sibling ports are forced to be closed */
2491                         ports[sibling].port_status = RTE_PORT_CLOSED;
2492                         printf("Port %u is closed\n", sibling);
2493                 }
2494         }
2495
2496         remove_invalid_ports();
2497
2498         printf("Device of port %u is detached\n", port_id);
2499         printf("Now total ports is %d\n", nb_ports);
2500         printf("Done\n");
2501         return;
2502 }
2503
2504 void
2505 detach_device(char *identifier)
2506 {
2507         struct rte_dev_iterator iterator;
2508         struct rte_devargs da;
2509         portid_t port_id;
2510
2511         printf("Removing a device...\n");
2512
2513         memset(&da, 0, sizeof(da));
2514         if (rte_devargs_parsef(&da, "%s", identifier)) {
2515                 printf("cannot parse identifier\n");
2516                 if (da.args)
2517                         free(da.args);
2518                 return;
2519         }
2520
2521         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2522                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2523                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2524                                 printf("Port %u not stopped\n", port_id);
2525                                 return;
2526                         }
2527
2528                         /* sibling ports are forced to be closed */
2529                         if (ports[port_id].flow_list)
2530                                 port_flow_flush(port_id);
2531                         ports[port_id].port_status = RTE_PORT_CLOSED;
2532                         printf("Port %u is now closed\n", port_id);
2533                 }
2534         }
2535
2536         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2537                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2538                             da.name, da.bus->name);
2539                 return;
2540         }
2541
2542         remove_invalid_ports();
2543
2544         printf("Device %s is detached\n", identifier);
2545         printf("Now total ports is %d\n", nb_ports);
2546         printf("Done\n");
2547 }
2548
2549 void
2550 pmd_test_exit(void)
2551 {
2552         portid_t pt_id;
2553         int ret;
2554         int i;
2555
2556         if (test_done == 0)
2557                 stop_packet_forwarding();
2558
2559         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2560                 if (mempools[i]) {
2561                         if (mp_alloc_type == MP_ALLOC_ANON)
2562                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2563                                                      NULL);
2564                 }
2565         }
2566         if (ports != NULL) {
2567                 no_link_check = 1;
2568                 RTE_ETH_FOREACH_DEV(pt_id) {
2569                         printf("\nStopping port %d...\n", pt_id);
2570                         fflush(stdout);
2571                         stop_port(pt_id);
2572                 }
2573                 RTE_ETH_FOREACH_DEV(pt_id) {
2574                         printf("\nShutting down port %d...\n", pt_id);
2575                         fflush(stdout);
2576                         close_port(pt_id);
2577                 }
2578         }
2579
2580         if (hot_plug) {
2581                 ret = rte_dev_event_monitor_stop();
2582                 if (ret) {
2583                         RTE_LOG(ERR, EAL,
2584                                 "fail to stop device event monitor.");
2585                         return;
2586                 }
2587
2588                 ret = rte_dev_event_callback_unregister(NULL,
2589                         dev_event_callback, NULL);
2590                 if (ret < 0) {
2591                         RTE_LOG(ERR, EAL,
2592                                 "fail to unregister device event callback.\n");
2593                         return;
2594                 }
2595
2596                 ret = rte_dev_hotplug_handle_disable();
2597                 if (ret) {
2598                         RTE_LOG(ERR, EAL,
2599                                 "fail to disable hotplug handling.\n");
2600                         return;
2601                 }
2602         }
2603         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2604                 if (mempools[i])
2605                         rte_mempool_free(mempools[i]);
2606         }
2607
2608         printf("\nBye...\n");
2609 }
2610
2611 typedef void (*cmd_func_t)(void);
2612 struct pmd_test_command {
2613         const char *cmd_name;
2614         cmd_func_t cmd_func;
2615 };
2616
2617 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2618
2619 /* Check the link status of all ports in up to 9s, and print them finally */
2620 static void
2621 check_all_ports_link_status(uint32_t port_mask)
2622 {
2623 #define CHECK_INTERVAL 100 /* 100ms */
2624 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2625         portid_t portid;
2626         uint8_t count, all_ports_up, print_flag = 0;
2627         struct rte_eth_link link;
2628
2629         printf("Checking link statuses...\n");
2630         fflush(stdout);
2631         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2632                 all_ports_up = 1;
2633                 RTE_ETH_FOREACH_DEV(portid) {
2634                         if ((port_mask & (1 << portid)) == 0)
2635                                 continue;
2636                         memset(&link, 0, sizeof(link));
2637                         rte_eth_link_get_nowait(portid, &link);
2638                         /* print link status if flag set */
2639                         if (print_flag == 1) {
2640                                 if (link.link_status)
2641                                         printf(
2642                                         "Port%d Link Up. speed %u Mbps- %s\n",
2643                                         portid, link.link_speed,
2644                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2645                                         ("full-duplex") : ("half-duplex\n"));
2646                                 else
2647                                         printf("Port %d Link Down\n", portid);
2648                                 continue;
2649                         }
2650                         /* clear all_ports_up flag if any link down */
2651                         if (link.link_status == ETH_LINK_DOWN) {
2652                                 all_ports_up = 0;
2653                                 break;
2654                         }
2655                 }
2656                 /* after finally printing all link status, get out */
2657                 if (print_flag == 1)
2658                         break;
2659
2660                 if (all_ports_up == 0) {
2661                         fflush(stdout);
2662                         rte_delay_ms(CHECK_INTERVAL);
2663                 }
2664
2665                 /* set the print_flag if all ports up or timeout */
2666                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2667                         print_flag = 1;
2668                 }
2669
2670                 if (lsc_interrupt)
2671                         break;
2672         }
2673 }
2674
2675 /*
2676  * This callback is for remove a port for a device. It has limitation because
2677  * it is not for multiple port removal for a device.
2678  * TODO: the device detach invoke will plan to be removed from user side to
2679  * eal. And convert all PMDs to free port resources on ether device closing.
2680  */
2681 static void
2682 rmv_port_callback(void *arg)
2683 {
2684         int need_to_start = 0;
2685         int org_no_link_check = no_link_check;
2686         portid_t port_id = (intptr_t)arg;
2687
2688         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2689
2690         if (!test_done && port_is_forwarding(port_id)) {
2691                 need_to_start = 1;
2692                 stop_packet_forwarding();
2693         }
2694         no_link_check = 1;
2695         stop_port(port_id);
2696         no_link_check = org_no_link_check;
2697         close_port(port_id);
2698         detach_port_device(port_id);
2699         if (need_to_start)
2700                 start_packet_forwarding(0);
2701 }
2702
2703 /* This function is used by the interrupt thread */
2704 static int
2705 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2706                   void *ret_param)
2707 {
2708         RTE_SET_USED(param);
2709         RTE_SET_USED(ret_param);
2710
2711         if (type >= RTE_ETH_EVENT_MAX) {
2712                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2713                         port_id, __func__, type);
2714                 fflush(stderr);
2715         } else if (event_print_mask & (UINT32_C(1) << type)) {
2716                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2717                         eth_event_desc[type]);
2718                 fflush(stdout);
2719         }
2720
2721         switch (type) {
2722         case RTE_ETH_EVENT_NEW:
2723                 ports[port_id].need_setup = 1;
2724                 ports[port_id].port_status = RTE_PORT_HANDLING;
2725                 break;
2726         case RTE_ETH_EVENT_INTR_RMV:
2727                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2728                         break;
2729                 if (rte_eal_alarm_set(100000,
2730                                 rmv_port_callback, (void *)(intptr_t)port_id))
2731                         fprintf(stderr, "Could not set up deferred device removal\n");
2732                 break;
2733         default:
2734                 break;
2735         }
2736         return 0;
2737 }
2738
2739 static int
2740 register_eth_event_callback(void)
2741 {
2742         int ret;
2743         enum rte_eth_event_type event;
2744
2745         for (event = RTE_ETH_EVENT_UNKNOWN;
2746                         event < RTE_ETH_EVENT_MAX; event++) {
2747                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2748                                 event,
2749                                 eth_event_callback,
2750                                 NULL);
2751                 if (ret != 0) {
2752                         TESTPMD_LOG(ERR, "Failed to register callback for "
2753                                         "%s event\n", eth_event_desc[event]);
2754                         return -1;
2755                 }
2756         }
2757
2758         return 0;
2759 }
2760
2761 /* This function is used by the interrupt thread */
2762 static void
2763 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2764                              __rte_unused void *arg)
2765 {
2766         uint16_t port_id;
2767         int ret;
2768
2769         if (type >= RTE_DEV_EVENT_MAX) {
2770                 fprintf(stderr, "%s called upon invalid event %d\n",
2771                         __func__, type);
2772                 fflush(stderr);
2773         }
2774
2775         switch (type) {
2776         case RTE_DEV_EVENT_REMOVE:
2777                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2778                         device_name);
2779                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2780                 if (ret) {
2781                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2782                                 device_name);
2783                         return;
2784                 }
2785                 /*
2786                  * Because the user's callback is invoked in eal interrupt
2787                  * callback, the interrupt callback need to be finished before
2788                  * it can be unregistered when detaching device. So finish
2789                  * callback soon and use a deferred removal to detach device
2790                  * is need. It is a workaround, once the device detaching be
2791                  * moved into the eal in the future, the deferred removal could
2792                  * be deleted.
2793                  */
2794                 if (rte_eal_alarm_set(100000,
2795                                 rmv_port_callback, (void *)(intptr_t)port_id))
2796                         RTE_LOG(ERR, EAL,
2797                                 "Could not set up deferred device removal\n");
2798                 break;
2799         case RTE_DEV_EVENT_ADD:
2800                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2801                         device_name);
2802                 /* TODO: After finish kernel driver binding,
2803                  * begin to attach port.
2804                  */
2805                 break;
2806         default:
2807                 break;
2808         }
2809 }
2810
2811 static int
2812 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2813 {
2814         uint16_t i;
2815         int diag;
2816         uint8_t mapping_found = 0;
2817
2818         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2819                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2820                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2821                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2822                                         tx_queue_stats_mappings[i].queue_id,
2823                                         tx_queue_stats_mappings[i].stats_counter_id);
2824                         if (diag != 0)
2825                                 return diag;
2826                         mapping_found = 1;
2827                 }
2828         }
2829         if (mapping_found)
2830                 port->tx_queue_stats_mapping_enabled = 1;
2831         return 0;
2832 }
2833
2834 static int
2835 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2836 {
2837         uint16_t i;
2838         int diag;
2839         uint8_t mapping_found = 0;
2840
2841         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2842                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2843                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2844                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2845                                         rx_queue_stats_mappings[i].queue_id,
2846                                         rx_queue_stats_mappings[i].stats_counter_id);
2847                         if (diag != 0)
2848                                 return diag;
2849                         mapping_found = 1;
2850                 }
2851         }
2852         if (mapping_found)
2853                 port->rx_queue_stats_mapping_enabled = 1;
2854         return 0;
2855 }
2856
2857 static void
2858 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2859 {
2860         int diag = 0;
2861
2862         diag = set_tx_queue_stats_mapping_registers(pi, port);
2863         if (diag != 0) {
2864                 if (diag == -ENOTSUP) {
2865                         port->tx_queue_stats_mapping_enabled = 0;
2866                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2867                 }
2868                 else
2869                         rte_exit(EXIT_FAILURE,
2870                                         "set_tx_queue_stats_mapping_registers "
2871                                         "failed for port id=%d diag=%d\n",
2872                                         pi, diag);
2873         }
2874
2875         diag = set_rx_queue_stats_mapping_registers(pi, port);
2876         if (diag != 0) {
2877                 if (diag == -ENOTSUP) {
2878                         port->rx_queue_stats_mapping_enabled = 0;
2879                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2880                 }
2881                 else
2882                         rte_exit(EXIT_FAILURE,
2883                                         "set_rx_queue_stats_mapping_registers "
2884                                         "failed for port id=%d diag=%d\n",
2885                                         pi, diag);
2886         }
2887 }
2888
2889 static void
2890 rxtx_port_config(struct rte_port *port)
2891 {
2892         uint16_t qid;
2893         uint64_t offloads;
2894
2895         for (qid = 0; qid < nb_rxq; qid++) {
2896                 offloads = port->rx_conf[qid].offloads;
2897                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2898                 if (offloads != 0)
2899                         port->rx_conf[qid].offloads = offloads;
2900
2901                 /* Check if any Rx parameters have been passed */
2902                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2903                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2904
2905                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2906                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2907
2908                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2909                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2910
2911                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2912                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2913
2914                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2915                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2916
2917                 port->nb_rx_desc[qid] = nb_rxd;
2918         }
2919
2920         for (qid = 0; qid < nb_txq; qid++) {
2921                 offloads = port->tx_conf[qid].offloads;
2922                 port->tx_conf[qid] = port->dev_info.default_txconf;
2923                 if (offloads != 0)
2924                         port->tx_conf[qid].offloads = offloads;
2925
2926                 /* Check if any Tx parameters have been passed */
2927                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2928                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2929
2930                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2931                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2932
2933                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2934                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2935
2936                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2937                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2938
2939                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2940                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2941
2942                 port->nb_tx_desc[qid] = nb_txd;
2943         }
2944 }
2945
2946 void
2947 init_port_config(void)
2948 {
2949         portid_t pid;
2950         struct rte_port *port;
2951         int ret;
2952
2953         RTE_ETH_FOREACH_DEV(pid) {
2954                 port = &ports[pid];
2955                 port->dev_conf.fdir_conf = fdir_conf;
2956
2957                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
2958                 if (ret != 0)
2959                         return;
2960
2961                 if (nb_rxq > 1) {
2962                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2963                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2964                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2965                 } else {
2966                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2967                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2968                 }
2969
2970                 if (port->dcb_flag == 0) {
2971                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2972                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2973                         else
2974                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2975                 }
2976
2977                 rxtx_port_config(port);
2978
2979                 rte_eth_macaddr_get(pid, &port->eth_addr);
2980
2981                 map_port_queue_stats_mapping_registers(pid, port);
2982 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2983                 rte_pmd_ixgbe_bypass_init(pid);
2984 #endif
2985
2986                 if (lsc_interrupt &&
2987                     (rte_eth_devices[pid].data->dev_flags &
2988                      RTE_ETH_DEV_INTR_LSC))
2989                         port->dev_conf.intr_conf.lsc = 1;
2990                 if (rmv_interrupt &&
2991                     (rte_eth_devices[pid].data->dev_flags &
2992                      RTE_ETH_DEV_INTR_RMV))
2993                         port->dev_conf.intr_conf.rmv = 1;
2994         }
2995 }
2996
2997 void set_port_slave_flag(portid_t slave_pid)
2998 {
2999         struct rte_port *port;
3000
3001         port = &ports[slave_pid];
3002         port->slave_flag = 1;
3003 }
3004
3005 void clear_port_slave_flag(portid_t slave_pid)
3006 {
3007         struct rte_port *port;
3008
3009         port = &ports[slave_pid];
3010         port->slave_flag = 0;
3011 }
3012
3013 uint8_t port_is_bonding_slave(portid_t slave_pid)
3014 {
3015         struct rte_port *port;
3016
3017         port = &ports[slave_pid];
3018         if ((rte_eth_devices[slave_pid].data->dev_flags &
3019             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3020                 return 1;
3021         return 0;
3022 }
3023
3024 const uint16_t vlan_tags[] = {
3025                 0,  1,  2,  3,  4,  5,  6,  7,
3026                 8,  9, 10, 11,  12, 13, 14, 15,
3027                 16, 17, 18, 19, 20, 21, 22, 23,
3028                 24, 25, 26, 27, 28, 29, 30, 31
3029 };
3030
3031 static  int
3032 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3033                  enum dcb_mode_enable dcb_mode,
3034                  enum rte_eth_nb_tcs num_tcs,
3035                  uint8_t pfc_en)
3036 {
3037         uint8_t i;
3038         int32_t rc;
3039         struct rte_eth_rss_conf rss_conf;
3040
3041         /*
3042          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3043          * given above, and the number of traffic classes available for use.
3044          */
3045         if (dcb_mode == DCB_VT_ENABLED) {
3046                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3047                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3048                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3049                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3050
3051                 /* VMDQ+DCB RX and TX configurations */
3052                 vmdq_rx_conf->enable_default_pool = 0;
3053                 vmdq_rx_conf->default_pool = 0;
3054                 vmdq_rx_conf->nb_queue_pools =
3055                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3056                 vmdq_tx_conf->nb_queue_pools =
3057                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3058
3059                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3060                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3061                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3062                         vmdq_rx_conf->pool_map[i].pools =
3063                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3064                 }
3065                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3066                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3067                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3068                 }
3069
3070                 /* set DCB mode of RX and TX of multiple queues */
3071                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
3072                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3073         } else {
3074                 struct rte_eth_dcb_rx_conf *rx_conf =
3075                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3076                 struct rte_eth_dcb_tx_conf *tx_conf =
3077                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3078
3079                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3080                 if (rc != 0)
3081                         return rc;
3082
3083                 rx_conf->nb_tcs = num_tcs;
3084                 tx_conf->nb_tcs = num_tcs;
3085
3086                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3087                         rx_conf->dcb_tc[i] = i % num_tcs;
3088                         tx_conf->dcb_tc[i] = i % num_tcs;
3089                 }
3090
3091                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
3092                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3093                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3094         }
3095
3096         if (pfc_en)
3097                 eth_conf->dcb_capability_en =
3098                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3099         else
3100                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3101
3102         return 0;
3103 }
3104
3105 int
3106 init_port_dcb_config(portid_t pid,
3107                      enum dcb_mode_enable dcb_mode,
3108                      enum rte_eth_nb_tcs num_tcs,
3109                      uint8_t pfc_en)
3110 {
3111         struct rte_eth_conf port_conf;
3112         struct rte_port *rte_port;
3113         int retval;
3114         uint16_t i;
3115
3116         rte_port = &ports[pid];
3117
3118         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3119         /* Enter DCB configuration status */
3120         dcb_config = 1;
3121
3122         port_conf.rxmode = rte_port->dev_conf.rxmode;
3123         port_conf.txmode = rte_port->dev_conf.txmode;
3124
3125         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3126         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3127         if (retval < 0)
3128                 return retval;
3129         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3130
3131         /* re-configure the device . */
3132         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3133         if (retval < 0)
3134                 return retval;
3135
3136         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3137         if (retval != 0)
3138                 return retval;
3139
3140         /* If dev_info.vmdq_pool_base is greater than 0,
3141          * the queue id of vmdq pools is started after pf queues.
3142          */
3143         if (dcb_mode == DCB_VT_ENABLED &&
3144             rte_port->dev_info.vmdq_pool_base > 0) {
3145                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3146                         " for port %d.", pid);
3147                 return -1;
3148         }
3149
3150         /* Assume the ports in testpmd have the same dcb capability
3151          * and has the same number of rxq and txq in dcb mode
3152          */
3153         if (dcb_mode == DCB_VT_ENABLED) {
3154                 if (rte_port->dev_info.max_vfs > 0) {
3155                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3156                         nb_txq = rte_port->dev_info.nb_tx_queues;
3157                 } else {
3158                         nb_rxq = rte_port->dev_info.max_rx_queues;
3159                         nb_txq = rte_port->dev_info.max_tx_queues;
3160                 }
3161         } else {
3162                 /*if vt is disabled, use all pf queues */
3163                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3164                         nb_rxq = rte_port->dev_info.max_rx_queues;
3165                         nb_txq = rte_port->dev_info.max_tx_queues;
3166                 } else {
3167                         nb_rxq = (queueid_t)num_tcs;
3168                         nb_txq = (queueid_t)num_tcs;
3169
3170                 }
3171         }
3172         rx_free_thresh = 64;
3173
3174         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3175
3176         rxtx_port_config(rte_port);
3177         /* VLAN filter */
3178         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3179         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3180                 rx_vft_set(pid, vlan_tags[i], 1);
3181
3182         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
3183         map_port_queue_stats_mapping_registers(pid, rte_port);
3184
3185         rte_port->dcb_flag = 1;
3186
3187         return 0;
3188 }
3189
3190 static void
3191 init_port(void)
3192 {
3193         /* Configuration of Ethernet ports. */
3194         ports = rte_zmalloc("testpmd: ports",
3195                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3196                             RTE_CACHE_LINE_SIZE);
3197         if (ports == NULL) {
3198                 rte_exit(EXIT_FAILURE,
3199                                 "rte_zmalloc(%d struct rte_port) failed\n",
3200                                 RTE_MAX_ETHPORTS);
3201         }
3202
3203         /* Initialize ports NUMA structures */
3204         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3205         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3206         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3207 }
3208
3209 static void
3210 force_quit(void)
3211 {
3212         pmd_test_exit();
3213         prompt_exit();
3214 }
3215
3216 static void
3217 print_stats(void)
3218 {
3219         uint8_t i;
3220         const char clr[] = { 27, '[', '2', 'J', '\0' };
3221         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3222
3223         /* Clear screen and move to top left */
3224         printf("%s%s", clr, top_left);
3225
3226         printf("\nPort statistics ====================================");
3227         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3228                 nic_stats_display(fwd_ports_ids[i]);
3229
3230         fflush(stdout);
3231 }
3232
3233 static void
3234 signal_handler(int signum)
3235 {
3236         if (signum == SIGINT || signum == SIGTERM) {
3237                 printf("\nSignal %d received, preparing to exit...\n",
3238                                 signum);
3239 #ifdef RTE_LIBRTE_PDUMP
3240                 /* uninitialize packet capture framework */
3241                 rte_pdump_uninit();
3242 #endif
3243 #ifdef RTE_LIBRTE_LATENCY_STATS
3244                 if (latencystats_enabled != 0)
3245                         rte_latencystats_uninit();
3246 #endif
3247                 force_quit();
3248                 /* Set flag to indicate the force termination. */
3249                 f_quit = 1;
3250                 /* exit with the expected status */
3251                 signal(signum, SIG_DFL);
3252                 kill(getpid(), signum);
3253         }
3254 }
3255
3256 int
3257 main(int argc, char** argv)
3258 {
3259         int diag;
3260         portid_t port_id;
3261         uint16_t count;
3262         int ret;
3263
3264         signal(SIGINT, signal_handler);
3265         signal(SIGTERM, signal_handler);
3266
3267         testpmd_logtype = rte_log_register("testpmd");
3268         if (testpmd_logtype < 0)
3269                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3270         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3271
3272         diag = rte_eal_init(argc, argv);
3273         if (diag < 0)
3274                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3275                          rte_strerror(rte_errno));
3276
3277         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3278                 rte_exit(EXIT_FAILURE,
3279                          "Secondary process type not supported.\n");
3280
3281         ret = register_eth_event_callback();
3282         if (ret != 0)
3283                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3284
3285 #ifdef RTE_LIBRTE_PDUMP
3286         /* initialize packet capture framework */
3287         rte_pdump_init();
3288 #endif
3289
3290         count = 0;
3291         RTE_ETH_FOREACH_DEV(port_id) {
3292                 ports_ids[count] = port_id;
3293                 count++;
3294         }
3295         nb_ports = (portid_t) count;
3296         if (nb_ports == 0)
3297                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3298
3299         /* allocate port structures, and init them */
3300         init_port();
3301
3302         set_def_fwd_config();
3303         if (nb_lcores == 0)
3304                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3305                          "Check the core mask argument\n");
3306
3307         /* Bitrate/latency stats disabled by default */
3308 #ifdef RTE_LIBRTE_BITRATE
3309         bitrate_enabled = 0;
3310 #endif
3311 #ifdef RTE_LIBRTE_LATENCY_STATS
3312         latencystats_enabled = 0;
3313 #endif
3314
3315         /* on FreeBSD, mlockall() is disabled by default */
3316 #ifdef RTE_EXEC_ENV_FREEBSD
3317         do_mlockall = 0;
3318 #else
3319         do_mlockall = 1;
3320 #endif
3321
3322         argc -= diag;
3323         argv += diag;
3324         if (argc > 1)
3325                 launch_args_parse(argc, argv);
3326
3327         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3328                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3329                         strerror(errno));
3330         }
3331
3332         if (tx_first && interactive)
3333                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3334                                 "interactive mode.\n");
3335
3336         if (tx_first && lsc_interrupt) {
3337                 printf("Warning: lsc_interrupt needs to be off when "
3338                                 " using tx_first. Disabling.\n");
3339                 lsc_interrupt = 0;
3340         }
3341
3342         if (!nb_rxq && !nb_txq)
3343                 printf("Warning: Either rx or tx queues should be non-zero\n");
3344
3345         if (nb_rxq > 1 && nb_rxq > nb_txq)
3346                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3347                        "but nb_txq=%d will prevent to fully test it.\n",
3348                        nb_rxq, nb_txq);
3349
3350         init_config();
3351
3352         if (hot_plug) {
3353                 ret = rte_dev_hotplug_handle_enable();
3354                 if (ret) {
3355                         RTE_LOG(ERR, EAL,
3356                                 "fail to enable hotplug handling.");
3357                         return -1;
3358                 }
3359
3360                 ret = rte_dev_event_monitor_start();
3361                 if (ret) {
3362                         RTE_LOG(ERR, EAL,
3363                                 "fail to start device event monitoring.");
3364                         return -1;
3365                 }
3366
3367                 ret = rte_dev_event_callback_register(NULL,
3368                         dev_event_callback, NULL);
3369                 if (ret) {
3370                         RTE_LOG(ERR, EAL,
3371                                 "fail  to register device event callback\n");
3372                         return -1;
3373                 }
3374         }
3375
3376         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3377                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3378
3379         /* set all ports to promiscuous mode by default */
3380         RTE_ETH_FOREACH_DEV(port_id) {
3381                 ret = rte_eth_promiscuous_enable(port_id);
3382                 if (ret != 0)
3383                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3384                                 port_id, rte_strerror(-ret));
3385         }
3386
3387         /* Init metrics library */
3388         rte_metrics_init(rte_socket_id());
3389
3390 #ifdef RTE_LIBRTE_LATENCY_STATS
3391         if (latencystats_enabled != 0) {
3392                 int ret = rte_latencystats_init(1, NULL);
3393                 if (ret)
3394                         printf("Warning: latencystats init()"
3395                                 " returned error %d\n", ret);
3396                 printf("Latencystats running on lcore %d\n",
3397                         latencystats_lcore_id);
3398         }
3399 #endif
3400
3401         /* Setup bitrate stats */
3402 #ifdef RTE_LIBRTE_BITRATE
3403         if (bitrate_enabled != 0) {
3404                 bitrate_data = rte_stats_bitrate_create();
3405                 if (bitrate_data == NULL)
3406                         rte_exit(EXIT_FAILURE,
3407                                 "Could not allocate bitrate data.\n");
3408                 rte_stats_bitrate_reg(bitrate_data);
3409         }
3410 #endif
3411
3412 #ifdef RTE_LIBRTE_CMDLINE
3413         if (strlen(cmdline_filename) != 0)
3414                 cmdline_read_from_file(cmdline_filename);
3415
3416         if (interactive == 1) {
3417                 if (auto_start) {
3418                         printf("Start automatic packet forwarding\n");
3419                         start_packet_forwarding(0);
3420                 }
3421                 prompt();
3422                 pmd_test_exit();
3423         } else
3424 #endif
3425         {
3426                 char c;
3427                 int rc;
3428
3429                 f_quit = 0;
3430
3431                 printf("No commandline core given, start packet forwarding\n");
3432                 start_packet_forwarding(tx_first);
3433                 if (stats_period != 0) {
3434                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3435                         uint64_t timer_period;
3436
3437                         /* Convert to number of cycles */
3438                         timer_period = stats_period * rte_get_timer_hz();
3439
3440                         while (f_quit == 0) {
3441                                 cur_time = rte_get_timer_cycles();
3442                                 diff_time += cur_time - prev_time;
3443
3444                                 if (diff_time >= timer_period) {
3445                                         print_stats();
3446                                         /* Reset the timer */
3447                                         diff_time = 0;
3448                                 }
3449                                 /* Sleep to avoid unnecessary checks */
3450                                 prev_time = cur_time;
3451                                 sleep(1);
3452                         }
3453                 }
3454
3455                 printf("Press enter to exit\n");
3456                 rc = read(0, &c, 1);
3457                 pmd_test_exit();
3458                 if (rc < 0)
3459                         return 1;
3460         }
3461
3462         return 0;
3463 }