app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_malloc_heap.h>
  31 #include <rte_memory.h>
  32 #include <rte_memcpy.h>
  33 #include <rte_launch.h>
  34 #include <rte_eal.h>
  35 #include <rte_alarm.h>
  36 #include <rte_per_lcore.h>
  37 #include <rte_lcore.h>
  38 #include <rte_atomic.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_LIBRTE_IXGBE_PMD
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIBRTE_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #include <rte_metrics.h>
  58 #ifdef RTE_LIBRTE_BITRATE
  59 #include <rte_bitrate.h>
  60 #endif
  61 #ifdef RTE_LIBRTE_LATENCY_STATS
  62 #include <rte_latencystats.h>
  63 #endif
  64
  65 #include "testpmd.h"
  66
  67 #ifndef MAP_HUGETLB
  68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  69 #define HUGE_FLAG (0x40000)
  70 #else
  71 #define HUGE_FLAG MAP_HUGETLB
  72 #endif
  73
  74 #ifndef MAP_HUGE_SHIFT
  75 /* older kernels (or FreeBSD) will not have this define */
  76 #define HUGE_SHIFT (26)
  77 #else
  78 #define HUGE_SHIFT MAP_HUGE_SHIFT
  79 #endif
  80
  81 #define EXTMEM_HEAP_NAME "extmem"
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192 uint16_t mempool_flags;
 193
 194 struct fwd_config cur_fwd_config;
 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 196 uint32_t retry_enabled;
 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 199
 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 201 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 202                                       * specified on command-line. */
 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 204
 205 /*
 206  * In container, it cannot terminate the process which running with 'stats-period'
 207  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 208  */
 209 uint8_t f_quit;
 210
 211 /*
 212  * Configuration of packet segments used by the "txonly" processing engine.
 213  */
 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 216         TXONLY_DEF_PACKET_LEN,
 217 };
 218 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 219
 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 221 /**< Split policy for packets to TX. */
 222
 223 uint8_t txonly_multi_flow;
 224 /**< Whether multiple flows are generated in TXONLY mode. */
 225
 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 228
 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 230 uint8_t dcb_config = 0;
 231
 232 /* Whether the dcb is in testing status */
 233 uint8_t dcb_test = 0;
 234
 235 /*
 236  * Configurable number of RX/TX queues.
 237  */
 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 240
 241 /*
 242  * Configurable number of RX/TX ring descriptors.
 243  * Defaults are supplied by drivers via ethdev.
 244  */
 245 #define RTE_TEST_RX_DESC_DEFAULT 0
 246 #define RTE_TEST_TX_DESC_DEFAULT 0
 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 249
 250 #define RTE_PMD_PARAM_UNSET -1
 251 /*
 252  * Configurable values of RX and TX ring threshold registers.
 253  */
 254
 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 258
 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 262
 263 /*
 264  * Configurable value of RX free threshold.
 265  */
 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 267
 268 /*
 269  * Configurable value of RX drop enable.
 270  */
 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 272
 273 /*
 274  * Configurable value of TX free threshold.
 275  */
 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of TX RS bit threshold.
 280  */
 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of buffered packets before sending.
 285  */
 286 uint16_t noisy_tx_sw_bufsz;
 287
 288 /*
 289  * Configurable value of packet buffer timeout.
 290  */
 291 uint16_t noisy_tx_sw_buf_flush_time;
 292
 293 /*
 294  * Configurable value for size of VNF internal memory area
 295  * used for simulating noisy neighbour behaviour
 296  */
 297 uint64_t noisy_lkup_mem_sz;
 298
 299 /*
 300  * Configurable value of number of random writes done in
 301  * VNF simulation memory area.
 302  */
 303 uint64_t noisy_lkup_num_writes;
 304
 305 /*
 306  * Configurable value of number of random reads done in
 307  * VNF simulation memory area.
 308  */
 309 uint64_t noisy_lkup_num_reads;
 310
 311 /*
 312  * Configurable value of number of random reads/writes done in
 313  * VNF simulation memory area.
 314  */
 315 uint64_t noisy_lkup_num_reads_writes;
 316
 317 /*
 318  * Receive Side Scaling (RSS) configuration.
 319  */
 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 321
 322 /*
 323  * Port topology configuration
 324  */
 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 326
 327 /*
 328  * Avoids to flush all the RX streams before starts forwarding.
 329  */
 330 uint8_t no_flush_rx = 0; /* flush by default */
 331
 332 /*
 333  * Flow API isolated mode.
 334  */
 335 uint8_t flow_isolate_all;
 336
 337 /*
 338  * Avoids to check link status when starting/stopping a port.
 339  */
 340 uint8_t no_link_check = 0; /* check by default */
 341
 342 /*
 343  * Don't automatically start all ports in interactive mode.
 344  */
 345 uint8_t no_device_start = 0;
 346
 347 /*
 348  * Enable link status change notification
 349  */
 350 uint8_t lsc_interrupt = 1; /* enabled by default */
 351
 352 /*
 353  * Enable device removal notification.
 354  */
 355 uint8_t rmv_interrupt = 1; /* enabled by default */
 356
 357 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 358
 359 /* After attach, port setup is called on event or by iterator */
 360 bool setup_on_probe_event = true;
 361
 362 /* Pretty printing of ethdev events */
 363 static const char * const eth_event_desc[] = {
 364         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 365         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 366         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 367         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 368         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 369         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 370         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 371         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 372         [RTE_ETH_EVENT_NEW] = "device probed",
 373         [RTE_ETH_EVENT_DESTROY] = "device released",
 374         [RTE_ETH_EVENT_MAX] = NULL,
 375 };
 376
 377 /*
 378  * Display or mask ether events
 379  * Default to all events except VF_MBOX
 380  */
 381 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 382                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 383                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 384                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 385                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 386                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 387                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 388 /*
 389  * Decide if all memory are locked for performance.
 390  */
 391 int do_mlockall = 0;
 392
 393 /*
 394  * NIC bypass mode configuration options.
 395  */
 396
 397 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 398 /* The NIC bypass watchdog timeout. */
 399 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 400 #endif
 401
 402
 403 #ifdef RTE_LIBRTE_LATENCY_STATS
 404
 405 /*
 406  * Set when latency stats is enabled in the commandline
 407  */
 408 uint8_t latencystats_enabled;
 409
 410 /*
 411  * Lcore ID to serive latency statistics.
 412  */
 413 lcoreid_t latencystats_lcore_id = -1;
 414
 415 #endif
 416
 417 /*
 418  * Ethernet device configuration.
 419  */
 420 struct rte_eth_rxmode rx_mode = {
 421         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 422                 /**< Default maximum frame length. */
 423 };
 424
 425 struct rte_eth_txmode tx_mode = {
 426         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 427 };
 428
 429 struct rte_fdir_conf fdir_conf = {
 430         .mode = RTE_FDIR_MODE_NONE,
 431         .pballoc = RTE_FDIR_PBALLOC_64K,
 432         .status = RTE_FDIR_REPORT_STATUS,
 433         .mask = {
 434                 .vlan_tci_mask = 0xFFEF,
 435                 .ipv4_mask     = {
 436                         .src_ip = 0xFFFFFFFF,
 437                         .dst_ip = 0xFFFFFFFF,
 438                 },
 439                 .ipv6_mask     = {
 440                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 441                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 442                 },
 443                 .src_port_mask = 0xFFFF,
 444                 .dst_port_mask = 0xFFFF,
 445                 .mac_addr_byte_mask = 0xFF,
 446                 .tunnel_type_mask = 1,
 447                 .tunnel_id_mask = 0xFFFFFFFF,
 448         },
 449         .drop_queue = 127,
 450 };
 451
 452 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 453
 454 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 455 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 456
 457 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 458 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 459
 460 uint16_t nb_tx_queue_stats_mappings = 0;
 461 uint16_t nb_rx_queue_stats_mappings = 0;
 462
 463 /*
 464  * Display zero values by default for xstats
 465  */
 466 uint8_t xstats_hide_zero;
 467
 468 unsigned int num_sockets = 0;
 469 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 470
 471 #ifdef RTE_LIBRTE_BITRATE
 472 /* Bitrate statistics */
 473 struct rte_stats_bitrates *bitrate_data;
 474 lcoreid_t bitrate_lcore_id;
 475 uint8_t bitrate_enabled;
 476 #endif
 477
 478 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 479 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 480
 481 struct vxlan_encap_conf vxlan_encap_conf = {
 482         .select_ipv4 = 1,
 483         .select_vlan = 0,
 484         .select_tos_ttl = 0,
 485         .vni = "\x00\x00\x00",
 486         .udp_src = 0,
 487         .udp_dst = RTE_BE16(4789),
 488         .ipv4_src = RTE_IPV4(127, 0, 0, 1),
 489         .ipv4_dst = RTE_IPV4(255, 255, 255, 255),
 490         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 491                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 492         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 493                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 494         .vlan_tci = 0,
 495         .ip_tos = 0,
 496         .ip_ttl = 255,
 497         .eth_src = "\x00\x00\x00\x00\x00\x00",
 498         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 499 };
 500
 501 struct nvgre_encap_conf nvgre_encap_conf = {
 502         .select_ipv4 = 1,
 503         .select_vlan = 0,
 504         .tni = "\x00\x00\x00",
 505         .ipv4_src = RTE_IPV4(127, 0, 0, 1),
 506         .ipv4_dst = RTE_IPV4(255, 255, 255, 255),
 507         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 508                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 509         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 510                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 511         .vlan_tci = 0,
 512         .eth_src = "\x00\x00\x00\x00\x00\x00",
 513         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 514 };
 515
 516 /* Forward function declarations */
 517 static void setup_attached_port(portid_t pi);
 518 static void map_port_queue_stats_mapping_registers(portid_t pi,
 519                                                    struct rte_port *port);
 520 static void check_all_ports_link_status(uint32_t port_mask);
 521 static int eth_event_callback(portid_t port_id,
 522                               enum rte_eth_event_type type,
 523                               void *param, void *ret_param);
 524 static void dev_event_callback(const char *device_name,
 525                                 enum rte_dev_event_type type,
 526                                 void *param);
 527
 528 /*
 529  * Check if all the ports are started.
 530  * If yes, return positive value. If not, return zero.
 531  */
 532 static int all_ports_started(void);
 533
 534 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 535 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 536
 537 /*
 538  * Helper function to check if socket is already discovered.
 539  * If yes, return positive value. If not, return zero.
 540  */
 541 int
 542 new_socket_id(unsigned int socket_id)
 543 {
 544         unsigned int i;
 545
 546         for (i = 0; i < num_sockets; i++) {
 547                 if (socket_ids[i] == socket_id)
 548                         return 0;
 549         }
 550         return 1;
 551 }
 552
 553 /*
 554  * Setup default configuration.
 555  */
 556 static void
 557 set_default_fwd_lcores_config(void)
 558 {
 559         unsigned int i;
 560         unsigned int nb_lc;
 561         unsigned int sock_num;
 562
 563         nb_lc = 0;
 564         for (i = 0; i < RTE_MAX_LCORE; i++) {
 565                 if (!rte_lcore_is_enabled(i))
 566                         continue;
 567                 sock_num = rte_lcore_to_socket_id(i);
 568                 if (new_socket_id(sock_num)) {
 569                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 570                                 rte_exit(EXIT_FAILURE,
 571                                          "Total sockets greater than %u\n",
 572                                          RTE_MAX_NUMA_NODES);
 573                         }
 574                         socket_ids[num_sockets++] = sock_num;
 575                 }
 576                 if (i == rte_get_master_lcore())
 577                         continue;
 578                 fwd_lcores_cpuids[nb_lc++] = i;
 579         }
 580         nb_lcores = (lcoreid_t) nb_lc;
 581         nb_cfg_lcores = nb_lcores;
 582         nb_fwd_lcores = 1;
 583 }
 584
 585 static void
 586 set_def_peer_eth_addrs(void)
 587 {
 588         portid_t i;
 589
 590         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 591                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 592                 peer_eth_addrs[i].addr_bytes[5] = i;
 593         }
 594 }
 595
 596 static void
 597 set_default_fwd_ports_config(void)
 598 {
 599         portid_t pt_id;
 600         int i = 0;
 601
 602         RTE_ETH_FOREACH_DEV(pt_id) {
 603                 fwd_ports_ids[i++] = pt_id;
 604
 605                 /* Update sockets info according to the attached device */
 606                 int socket_id = rte_eth_dev_socket_id(pt_id);
 607                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 608                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 609                                 rte_exit(EXIT_FAILURE,
 610                                          "Total sockets greater than %u\n",
 611                                          RTE_MAX_NUMA_NODES);
 612                         }
 613                         socket_ids[num_sockets++] = socket_id;
 614                 }
 615         }
 616
 617         nb_cfg_ports = nb_ports;
 618         nb_fwd_ports = nb_ports;
 619 }
 620
 621 void
 622 set_def_fwd_config(void)
 623 {
 624         set_default_fwd_lcores_config();
 625         set_def_peer_eth_addrs();
 626         set_default_fwd_ports_config();
 627 }
 628
 629 /* extremely pessimistic estimation of memory required to create a mempool */
 630 static int
 631 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 632 {
 633         unsigned int n_pages, mbuf_per_pg, leftover;
 634         uint64_t total_mem, mbuf_mem, obj_sz;
 635
 636         /* there is no good way to predict how much space the mempool will
 637          * occupy because it will allocate chunks on the fly, and some of those
 638          * will come from default DPDK memory while some will come from our
 639          * external memory, so just assume 128MB will be enough for everyone.
 640          */
 641         uint64_t hdr_mem = 128 << 20;
 642
 643         /* account for possible non-contiguousness */
 644         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 645         if (obj_sz > pgsz) {
 646                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 647                 return -1;
 648         }
 649
 650         mbuf_per_pg = pgsz / obj_sz;
 651         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 652         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 653
 654         mbuf_mem = n_pages * pgsz;
 655
 656         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 657
 658         if (total_mem > SIZE_MAX) {
 659                 TESTPMD_LOG(ERR, "Memory size too big\n");
 660                 return -1;
 661         }
 662         *out = (size_t)total_mem;
 663
 664         return 0;
 665 }
 666
 667 static int
 668 pagesz_flags(uint64_t page_sz)
 669 {
 670         /* as per mmap() manpage, all page sizes are log2 of page size
 671          * shifted by MAP_HUGE_SHIFT
 672          */
 673         int log2 = rte_log2_u64(page_sz);
 674
 675         return (log2 << HUGE_SHIFT);
 676 }
 677
 678 static void *
 679 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 680 {
 681         void *addr;
 682         int flags;
 683
 684         /* allocate anonymous hugepages */
 685         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 686         if (huge)
 687                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 688
 689         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 690         if (addr == MAP_FAILED)
 691                 return NULL;
 692
 693         return addr;
 694 }
 695
 696 struct extmem_param {
 697         void *addr;
 698         size_t len;
 699         size_t pgsz;
 700         rte_iova_t *iova_table;
 701         unsigned int iova_table_len;
 702 };
 703
 704 static int
 705 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 706                 bool huge)
 707 {
 708         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 709                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 710         unsigned int cur_page, n_pages, pgsz_idx;
 711         size_t mem_sz, cur_pgsz;
 712         rte_iova_t *iovas = NULL;
 713         void *addr;
 714         int ret;
 715
 716         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 717                 /* skip anything that is too big */
 718                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 719                         continue;
 720
 721                 cur_pgsz = pgsizes[pgsz_idx];
 722
 723                 /* if we were told not to allocate hugepages, override */
 724                 if (!huge)
 725                         cur_pgsz = sysconf(_SC_PAGESIZE);
 726
 727                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 728                 if (ret < 0) {
 729                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 730                         return -1;
 731                 }
 732
 733                 /* allocate our memory */
 734                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 735
 736                 /* if we couldn't allocate memory with a specified page size,
 737                  * that doesn't mean we can't do it with other page sizes, so
 738                  * try another one.
 739                  */
 740                 if (addr == NULL)
 741                         continue;
 742
 743                 /* store IOVA addresses for every page in this memory area */
 744                 n_pages = mem_sz / cur_pgsz;
 745
 746                 iovas = malloc(sizeof(*iovas) * n_pages);
 747
 748                 if (iovas == NULL) {
 749                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 750                         goto fail;
 751                 }
 752                 /* lock memory if it's not huge pages */
 753                 if (!huge)
 754                         mlock(addr, mem_sz);
 755
 756                 /* populate IOVA addresses */
 757                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 758                         rte_iova_t iova;
 759                         size_t offset;
 760                         void *cur;
 761
 762                         offset = cur_pgsz * cur_page;
 763                         cur = RTE_PTR_ADD(addr, offset);
 764
 765                         /* touch the page before getting its IOVA */
 766                         *(volatile char *)cur = 0;
 767
 768                         iova = rte_mem_virt2iova(cur);
 769
 770                         iovas[cur_page] = iova;
 771                 }
 772
 773                 break;
 774         }
 775         /* if we couldn't allocate anything */
 776         if (iovas == NULL)
 777                 return -1;
 778
 779         param->addr = addr;
 780         param->len = mem_sz;
 781         param->pgsz = cur_pgsz;
 782         param->iova_table = iovas;
 783         param->iova_table_len = n_pages;
 784
 785         return 0;
 786 fail:
 787         if (iovas)
 788                 free(iovas);
 789         if (addr)
 790                 munmap(addr, mem_sz);
 791
 792         return -1;
 793 }
 794
 795 static int
 796 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 797 {
 798         struct extmem_param param;
 799         int socket_id, ret;
 800
 801         memset(&param, 0, sizeof(param));
 802
 803         /* check if our heap exists */
 804         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 805         if (socket_id < 0) {
 806                 /* create our heap */
 807                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 808                 if (ret < 0) {
 809                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 810                         return -1;
 811                 }
 812         }
 813
 814         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 815         if (ret < 0) {
 816                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 817                 return -1;
 818         }
 819
 820         /* we now have a valid memory area, so add it to heap */
 821         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 822                         param.addr, param.len, param.iova_table,
 823                         param.iova_table_len, param.pgsz);
 824
 825         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 826
 827         /* not needed any more */
 828         free(param.iova_table);
 829
 830         if (ret < 0) {
 831                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 832                 munmap(param.addr, param.len);
 833                 return -1;
 834         }
 835
 836         /* success */
 837
 838         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 839                         param.len >> 20);
 840
 841         return 0;
 842 }
 843 static void
 844 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 845              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 846 {
 847         uint16_t pid = 0;
 848         int ret;
 849
 850         RTE_ETH_FOREACH_DEV(pid) {
 851                 struct rte_eth_dev *dev =
 852                         &rte_eth_devices[pid];
 853
 854                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 855                                         memhdr->len);
 856                 if (ret) {
 857                         TESTPMD_LOG(DEBUG,
 858                                     "unable to DMA unmap addr 0x%p "
 859                                     "for device %s\n",
 860                                     memhdr->addr, dev->data->name);
 861                 }
 862         }
 863         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 864         if (ret) {
 865                 TESTPMD_LOG(DEBUG,
 866                             "unable to un-register addr 0x%p\n", memhdr->addr);
 867         }
 868 }
 869
 870 static void
 871 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 872            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 873 {
 874         uint16_t pid = 0;
 875         size_t page_size = sysconf(_SC_PAGESIZE);
 876         int ret;
 877
 878         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 879                                   page_size);
 880         if (ret) {
 881                 TESTPMD_LOG(DEBUG,
 882                             "unable to register addr 0x%p\n", memhdr->addr);
 883                 return;
 884         }
 885         RTE_ETH_FOREACH_DEV(pid) {
 886                 struct rte_eth_dev *dev =
 887                         &rte_eth_devices[pid];
 888
 889                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 890                                       memhdr->len);
 891                 if (ret) {
 892                         TESTPMD_LOG(DEBUG,
 893                                     "unable to DMA map addr 0x%p "
 894                                     "for device %s\n",
 895                                     memhdr->addr, dev->data->name);
 896                 }
 897         }
 898 }
 899
 900 /*
 901  * Configuration initialisation done once at init time.
 902  */
 903 static struct rte_mempool *
 904 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 905                  unsigned int socket_id)
 906 {
 907         char pool_name[RTE_MEMPOOL_NAMESIZE];
 908         struct rte_mempool *rte_mp = NULL;
 909         uint32_t mb_size;
 910
 911         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 912         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 913
 914         TESTPMD_LOG(INFO,
 915                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 916                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 917
 918         switch (mp_alloc_type) {
 919         case MP_ALLOC_NATIVE:
 920                 {
 921                         /* wrapper to rte_mempool_create() */
 922                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 923                                         rte_mbuf_best_mempool_ops());
 924                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 925                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 926                         break;
 927                 }
 928         case MP_ALLOC_ANON:
 929                 {
 930                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 931                                 mb_size, (unsigned int) mb_mempool_cache,
 932                                 sizeof(struct rte_pktmbuf_pool_private),
 933                                 socket_id, mempool_flags);
 934                         if (rte_mp == NULL)
 935                                 goto err;
 936
 937                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 938                                 rte_mempool_free(rte_mp);
 939                                 rte_mp = NULL;
 940                                 goto err;
 941                         }
 942                         rte_pktmbuf_pool_init(rte_mp, NULL);
 943                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 944                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 945                         break;
 946                 }
 947         case MP_ALLOC_XMEM:
 948         case MP_ALLOC_XMEM_HUGE:
 949                 {
 950                         int heap_socket;
 951                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 952
 953                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 954                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 955
 956                         heap_socket =
 957                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 958                         if (heap_socket < 0)
 959                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 960
 961                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 962                                         rte_mbuf_best_mempool_ops());
 963                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 964                                         mb_mempool_cache, 0, mbuf_seg_size,
 965                                         heap_socket);
 966                         break;
 967                 }
 968         default:
 969                 {
 970                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 971                 }
 972         }
 973
 974 err:
 975         if (rte_mp == NULL) {
 976                 rte_exit(EXIT_FAILURE,
 977                         "Creation of mbuf pool for socket %u failed: %s\n",
 978                         socket_id, rte_strerror(rte_errno));
 979         } else if (verbose_level > 0) {
 980                 rte_mempool_dump(stdout, rte_mp);
 981         }
 982         return rte_mp;
 983 }
 984
 985 /*
 986  * Check given socket id is valid or not with NUMA mode,
 987  * if valid, return 0, else return -1
 988  */
 989 static int
 990 check_socket_id(const unsigned int socket_id)
 991 {
 992         static int warning_once = 0;
 993
 994         if (new_socket_id(socket_id)) {
 995                 if (!warning_once && numa_support)
 996                         printf("Warning: NUMA should be configured manually by"
 997                                " using --port-numa-config and"
 998                                " --ring-numa-config parameters along with"
 999                                " --numa.\n");
1000                 warning_once = 1;
1001                 return -1;
1002         }
1003         return 0;
1004 }
1005
1006 /*
1007  * Get the allowed maximum number of RX queues.
1008  * *pid return the port id which has minimal value of
1009  * max_rx_queues in all ports.
1010  */
1011 queueid_t
1012 get_allowed_max_nb_rxq(portid_t *pid)
1013 {
1014         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
1015         bool max_rxq_valid = false;
1016         portid_t pi;
1017         struct rte_eth_dev_info dev_info;
1018
1019         RTE_ETH_FOREACH_DEV(pi) {
1020                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1021                         continue;
1022
1023                 max_rxq_valid = true;
1024                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1025                         allowed_max_rxq = dev_info.max_rx_queues;
1026                         *pid = pi;
1027                 }
1028         }
1029         return max_rxq_valid ? allowed_max_rxq : 0;
1030 }
1031
1032 /*
1033  * Check input rxq is valid or not.
1034  * If input rxq is not greater than any of maximum number
1035  * of RX queues of all ports, it is valid.
1036  * if valid, return 0, else return -1
1037  */
1038 int
1039 check_nb_rxq(queueid_t rxq)
1040 {
1041         queueid_t allowed_max_rxq;
1042         portid_t pid = 0;
1043
1044         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1045         if (rxq > allowed_max_rxq) {
1046                 printf("Fail: input rxq (%u) can't be greater "
1047                        "than max_rx_queues (%u) of port %u\n",
1048                        rxq,
1049                        allowed_max_rxq,
1050                        pid);
1051                 return -1;
1052         }
1053         return 0;
1054 }
1055
1056 /*
1057  * Get the allowed maximum number of TX queues.
1058  * *pid return the port id which has minimal value of
1059  * max_tx_queues in all ports.
1060  */
1061 queueid_t
1062 get_allowed_max_nb_txq(portid_t *pid)
1063 {
1064         queueid_t allowed_max_txq = MAX_QUEUE_ID;
1065         bool max_txq_valid = false;
1066         portid_t pi;
1067         struct rte_eth_dev_info dev_info;
1068
1069         RTE_ETH_FOREACH_DEV(pi) {
1070                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1071                         continue;
1072
1073                 max_txq_valid = true;
1074                 if (dev_info.max_tx_queues < allowed_max_txq) {
1075                         allowed_max_txq = dev_info.max_tx_queues;
1076                         *pid = pi;
1077                 }
1078         }
1079         return max_txq_valid ? allowed_max_txq : 0;
1080 }
1081
1082 /*
1083  * Check input txq is valid or not.
1084  * If input txq is not greater than any of maximum number
1085  * of TX queues of all ports, it is valid.
1086  * if valid, return 0, else return -1
1087  */
1088 int
1089 check_nb_txq(queueid_t txq)
1090 {
1091         queueid_t allowed_max_txq;
1092         portid_t pid = 0;
1093
1094         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1095         if (txq > allowed_max_txq) {
1096                 printf("Fail: input txq (%u) can't be greater "
1097                        "than max_tx_queues (%u) of port %u\n",
1098                        txq,
1099                        allowed_max_txq,
1100                        pid);
1101                 return -1;
1102         }
1103         return 0;
1104 }
1105
1106 static void
1107 init_config(void)
1108 {
1109         portid_t pid;
1110         struct rte_port *port;
1111         struct rte_mempool *mbp;
1112         unsigned int nb_mbuf_per_pool;
1113         lcoreid_t  lc_id;
1114         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1115         struct rte_gro_param gro_param;
1116         uint32_t gso_types;
1117         uint16_t data_size;
1118         bool warning = 0;
1119         int k;
1120         int ret;
1121
1122         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1123
1124         /* Configuration of logical cores. */
1125         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1126                                 sizeof(struct fwd_lcore *) * nb_lcores,
1127                                 RTE_CACHE_LINE_SIZE);
1128         if (fwd_lcores == NULL) {
1129                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1130                                                         "failed\n", nb_lcores);
1131         }
1132         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1133                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1134                                                sizeof(struct fwd_lcore),
1135                                                RTE_CACHE_LINE_SIZE);
1136                 if (fwd_lcores[lc_id] == NULL) {
1137                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1138                                                                 "failed\n");
1139                 }
1140                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1141         }
1142
1143         RTE_ETH_FOREACH_DEV(pid) {
1144                 port = &ports[pid];
1145                 /* Apply default TxRx configuration for all ports */
1146                 port->dev_conf.txmode = tx_mode;
1147                 port->dev_conf.rxmode = rx_mode;
1148
1149                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1150                 if (ret != 0)
1151                         rte_exit(EXIT_FAILURE,
1152                                  "rte_eth_dev_info_get() failed\n");
1153
1154                 if (!(port->dev_info.tx_offload_capa &
1155                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1156                         port->dev_conf.txmode.offloads &=
1157                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1158                 if (!(port->dev_info.tx_offload_capa &
1159                         DEV_TX_OFFLOAD_MATCH_METADATA))
1160                         port->dev_conf.txmode.offloads &=
1161                                 ~DEV_TX_OFFLOAD_MATCH_METADATA;
1162                 if (numa_support) {
1163                         if (port_numa[pid] != NUMA_NO_CONFIG)
1164                                 port_per_socket[port_numa[pid]]++;
1165                         else {
1166                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1167
1168                                 /*
1169                                  * if socket_id is invalid,
1170                                  * set to the first available socket.
1171                                  */
1172                                 if (check_socket_id(socket_id) < 0)
1173                                         socket_id = socket_ids[0];
1174                                 port_per_socket[socket_id]++;
1175                         }
1176                 }
1177
1178                 /* Apply Rx offloads configuration */
1179                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1180                         port->rx_conf[k].offloads =
1181                                 port->dev_conf.rxmode.offloads;
1182                 /* Apply Tx offloads configuration */
1183                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1184                         port->tx_conf[k].offloads =
1185                                 port->dev_conf.txmode.offloads;
1186
1187                 /* set flag to initialize port/queue */
1188                 port->need_reconfig = 1;
1189                 port->need_reconfig_queues = 1;
1190                 port->tx_metadata = 0;
1191
1192                 /* Check for maximum number of segments per MTU. Accordingly
1193                  * update the mbuf data size.
1194                  */
1195                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1196                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1197                         data_size = rx_mode.max_rx_pkt_len /
1198                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1199
1200                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1201                                                         mbuf_data_size) {
1202                                 mbuf_data_size = data_size +
1203                                                  RTE_PKTMBUF_HEADROOM;
1204                                 warning = 1;
1205                         }
1206                 }
1207         }
1208
1209         if (warning)
1210                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1211                             mbuf_data_size);
1212
1213         /*
1214          * Create pools of mbuf.
1215          * If NUMA support is disabled, create a single pool of mbuf in
1216          * socket 0 memory by default.
1217          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1218          *
1219          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1220          * nb_txd can be configured at run time.
1221          */
1222         if (param_total_num_mbufs)
1223                 nb_mbuf_per_pool = param_total_num_mbufs;
1224         else {
1225                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1226                         (nb_lcores * mb_mempool_cache) +
1227                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1228                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1229         }
1230
1231         if (numa_support) {
1232                 uint8_t i;
1233
1234                 for (i = 0; i < num_sockets; i++)
1235                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1236                                                        nb_mbuf_per_pool,
1237                                                        socket_ids[i]);
1238         } else {
1239                 if (socket_num == UMA_NO_CONFIG)
1240                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1241                                                        nb_mbuf_per_pool, 0);
1242                 else
1243                         mempools[socket_num] = mbuf_pool_create
1244                                                         (mbuf_data_size,
1245                                                          nb_mbuf_per_pool,
1246                                                          socket_num);
1247         }
1248
1249         init_port_config();
1250
1251         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1252                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1253         /*
1254          * Records which Mbuf pool to use by each logical core, if needed.
1255          */
1256         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1257                 mbp = mbuf_pool_find(
1258                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1259
1260                 if (mbp == NULL)
1261                         mbp = mbuf_pool_find(0);
1262                 fwd_lcores[lc_id]->mbp = mbp;
1263                 /* initialize GSO context */
1264                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1265                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1266                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1267                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1268                         RTE_ETHER_CRC_LEN;
1269                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1270         }
1271
1272         /* Configuration of packet forwarding streams. */
1273         if (init_fwd_streams() < 0)
1274                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1275
1276         fwd_config_setup();
1277
1278         /* create a gro context for each lcore */
1279         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1280         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1281         gro_param.max_item_per_flow = MAX_PKT_BURST;
1282         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1283                 gro_param.socket_id = rte_lcore_to_socket_id(
1284                                 fwd_lcores_cpuids[lc_id]);
1285                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1286                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1287                         rte_exit(EXIT_FAILURE,
1288                                         "rte_gro_ctx_create() failed\n");
1289                 }
1290         }
1291
1292 #if defined RTE_LIBRTE_PMD_SOFTNIC
1293         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1294                 RTE_ETH_FOREACH_DEV(pid) {
1295                         port = &ports[pid];
1296                         const char *driver = port->dev_info.driver_name;
1297
1298                         if (strcmp(driver, "net_softnic") == 0)
1299                                 port->softport.fwd_lcore_arg = fwd_lcores;
1300                 }
1301         }
1302 #endif
1303
1304 }
1305
1306
1307 void
1308 reconfig(portid_t new_port_id, unsigned socket_id)
1309 {
1310         struct rte_port *port;
1311         int ret;
1312
1313         /* Reconfiguration of Ethernet ports. */
1314         port = &ports[new_port_id];
1315
1316         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1317         if (ret != 0)
1318                 return;
1319
1320         /* set flag to initialize port/queue */
1321         port->need_reconfig = 1;
1322         port->need_reconfig_queues = 1;
1323         port->socket_id = socket_id;
1324
1325         init_port_config();
1326 }
1327
1328
1329 int
1330 init_fwd_streams(void)
1331 {
1332         portid_t pid;
1333         struct rte_port *port;
1334         streamid_t sm_id, nb_fwd_streams_new;
1335         queueid_t q;
1336
1337         /* set socket id according to numa or not */
1338         RTE_ETH_FOREACH_DEV(pid) {
1339                 port = &ports[pid];
1340                 if (nb_rxq > port->dev_info.max_rx_queues) {
1341                         printf("Fail: nb_rxq(%d) is greater than "
1342                                 "max_rx_queues(%d)\n", nb_rxq,
1343                                 port->dev_info.max_rx_queues);
1344                         return -1;
1345                 }
1346                 if (nb_txq > port->dev_info.max_tx_queues) {
1347                         printf("Fail: nb_txq(%d) is greater than "
1348                                 "max_tx_queues(%d)\n", nb_txq,
1349                                 port->dev_info.max_tx_queues);
1350                         return -1;
1351                 }
1352                 if (numa_support) {
1353                         if (port_numa[pid] != NUMA_NO_CONFIG)
1354                                 port->socket_id = port_numa[pid];
1355                         else {
1356                                 port->socket_id = rte_eth_dev_socket_id(pid);
1357
1358                                 /*
1359                                  * if socket_id is invalid,
1360                                  * set to the first available socket.
1361                                  */
1362                                 if (check_socket_id(port->socket_id) < 0)
1363                                         port->socket_id = socket_ids[0];
1364                         }
1365                 }
1366                 else {
1367                         if (socket_num == UMA_NO_CONFIG)
1368                                 port->socket_id = 0;
1369                         else
1370                                 port->socket_id = socket_num;
1371                 }
1372         }
1373
1374         q = RTE_MAX(nb_rxq, nb_txq);
1375         if (q == 0) {
1376                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1377                 return -1;
1378         }
1379         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1380         if (nb_fwd_streams_new == nb_fwd_streams)
1381                 return 0;
1382         /* clear the old */
1383         if (fwd_streams != NULL) {
1384                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1385                         if (fwd_streams[sm_id] == NULL)
1386                                 continue;
1387                         rte_free(fwd_streams[sm_id]);
1388                         fwd_streams[sm_id] = NULL;
1389                 }
1390                 rte_free(fwd_streams);
1391                 fwd_streams = NULL;
1392         }
1393
1394         /* init new */
1395         nb_fwd_streams = nb_fwd_streams_new;
1396         if (nb_fwd_streams) {
1397                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1398                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1399                         RTE_CACHE_LINE_SIZE);
1400                 if (fwd_streams == NULL)
1401                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1402                                  " (struct fwd_stream *)) failed\n",
1403                                  nb_fwd_streams);
1404
1405                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1406                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1407                                 " struct fwd_stream", sizeof(struct fwd_stream),
1408                                 RTE_CACHE_LINE_SIZE);
1409                         if (fwd_streams[sm_id] == NULL)
1410                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1411                                          "(struct fwd_stream) failed\n");
1412                 }
1413         }
1414
1415         return 0;
1416 }
1417
1418 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1419 static void
1420 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1421 {
1422         unsigned int total_burst;
1423         unsigned int nb_burst;
1424         unsigned int burst_stats[3];
1425         uint16_t pktnb_stats[3];
1426         uint16_t nb_pkt;
1427         int burst_percent[3];
1428
1429         /*
1430          * First compute the total number of packet bursts and the
1431          * two highest numbers of bursts of the same number of packets.
1432          */
1433         total_burst = 0;
1434         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1435         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1436         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1437                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1438                 if (nb_burst == 0)
1439                         continue;
1440                 total_burst += nb_burst;
1441                 if (nb_burst > burst_stats[0]) {
1442                         burst_stats[1] = burst_stats[0];
1443                         pktnb_stats[1] = pktnb_stats[0];
1444                         burst_stats[0] = nb_burst;
1445                         pktnb_stats[0] = nb_pkt;
1446                 } else if (nb_burst > burst_stats[1]) {
1447                         burst_stats[1] = nb_burst;
1448                         pktnb_stats[1] = nb_pkt;
1449                 }
1450         }
1451         if (total_burst == 0)
1452                 return;
1453         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1454         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1455                burst_percent[0], (int) pktnb_stats[0]);
1456         if (burst_stats[0] == total_burst) {
1457                 printf("]\n");
1458                 return;
1459         }
1460         if (burst_stats[0] + burst_stats[1] == total_burst) {
1461                 printf(" + %d%% of %d pkts]\n",
1462                        100 - burst_percent[0], pktnb_stats[1]);
1463                 return;
1464         }
1465         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1466         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1467         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1468                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1469                 return;
1470         }
1471         printf(" + %d%% of %d pkts + %d%% of others]\n",
1472                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1473 }
1474 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1475
1476 static void
1477 fwd_stream_stats_display(streamid_t stream_id)
1478 {
1479         struct fwd_stream *fs;
1480         static const char *fwd_top_stats_border = "-------";
1481
1482         fs = fwd_streams[stream_id];
1483         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1484             (fs->fwd_dropped == 0))
1485                 return;
1486         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1487                "TX Port=%2d/Queue=%2d %s\n",
1488                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1489                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1490         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1491                " TX-dropped: %-14"PRIu64,
1492                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1493
1494         /* if checksum mode */
1495         if (cur_fwd_eng == &csum_fwd_engine) {
1496                 printf("  RX- bad IP checksum: %-14"PRIu64
1497                        "  Rx- bad L4 checksum: %-14"PRIu64
1498                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1499                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1500                         fs->rx_bad_outer_l4_csum);
1501         } else {
1502                 printf("\n");
1503         }
1504
1505 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1506         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1507         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1508 #endif
1509 }
1510
1511 void
1512 fwd_stats_display(void)
1513 {
1514         static const char *fwd_stats_border = "----------------------";
1515         static const char *acc_stats_border = "+++++++++++++++";
1516         struct {
1517                 struct fwd_stream *rx_stream;
1518                 struct fwd_stream *tx_stream;
1519                 uint64_t tx_dropped;
1520                 uint64_t rx_bad_ip_csum;
1521                 uint64_t rx_bad_l4_csum;
1522                 uint64_t rx_bad_outer_l4_csum;
1523         } ports_stats[RTE_MAX_ETHPORTS];
1524         uint64_t total_rx_dropped = 0;
1525         uint64_t total_tx_dropped = 0;
1526         uint64_t total_rx_nombuf = 0;
1527         struct rte_eth_stats stats;
1528 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1529         uint64_t fwd_cycles = 0;
1530 #endif
1531         uint64_t total_recv = 0;
1532         uint64_t total_xmit = 0;
1533         struct rte_port *port;
1534         streamid_t sm_id;
1535         portid_t pt_id;
1536         int i;
1537
1538         memset(ports_stats, 0, sizeof(ports_stats));
1539
1540         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1541                 struct fwd_stream *fs = fwd_streams[sm_id];
1542
1543                 if (cur_fwd_config.nb_fwd_streams >
1544                     cur_fwd_config.nb_fwd_ports) {
1545                         fwd_stream_stats_display(sm_id);
1546                 } else {
1547                         ports_stats[fs->tx_port].tx_stream = fs;
1548                         ports_stats[fs->rx_port].rx_stream = fs;
1549                 }
1550
1551                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1552
1553                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1554                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1555                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1556                                 fs->rx_bad_outer_l4_csum;
1557
1558 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1559                 fwd_cycles += fs->core_cycles;
1560 #endif
1561         }
1562         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1563                 uint8_t j;
1564
1565                 pt_id = fwd_ports_ids[i];
1566                 port = &ports[pt_id];
1567
1568                 rte_eth_stats_get(pt_id, &stats);
1569                 stats.ipackets -= port->stats.ipackets;
1570                 stats.opackets -= port->stats.opackets;
1571                 stats.ibytes -= port->stats.ibytes;
1572                 stats.obytes -= port->stats.obytes;
1573                 stats.imissed -= port->stats.imissed;
1574                 stats.oerrors -= port->stats.oerrors;
1575                 stats.rx_nombuf -= port->stats.rx_nombuf;
1576
1577                 total_recv += stats.ipackets;
1578                 total_xmit += stats.opackets;
1579                 total_rx_dropped += stats.imissed;
1580                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1581                 total_tx_dropped += stats.oerrors;
1582                 total_rx_nombuf  += stats.rx_nombuf;
1583
1584                 printf("\n  %s Forward statistics for port %-2d %s\n",
1585                        fwd_stats_border, pt_id, fwd_stats_border);
1586
1587                 if (!port->rx_queue_stats_mapping_enabled &&
1588                     !port->tx_queue_stats_mapping_enabled) {
1589                         printf("  RX-packets: %-14"PRIu64
1590                                " RX-dropped: %-14"PRIu64
1591                                "RX-total: %-"PRIu64"\n",
1592                                stats.ipackets, stats.imissed,
1593                                stats.ipackets + stats.imissed);
1594
1595                         if (cur_fwd_eng == &csum_fwd_engine)
1596                                 printf("  Bad-ipcsum: %-14"PRIu64
1597                                        " Bad-l4csum: %-14"PRIu64
1598                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1599                                        ports_stats[pt_id].rx_bad_ip_csum,
1600                                        ports_stats[pt_id].rx_bad_l4_csum,
1601                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1602                         if (stats.ierrors + stats.rx_nombuf > 0) {
1603                                 printf("  RX-error: %-"PRIu64"\n",
1604                                        stats.ierrors);
1605                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1606                                        stats.rx_nombuf);
1607                         }
1608
1609                         printf("  TX-packets: %-14"PRIu64
1610                                " TX-dropped: %-14"PRIu64
1611                                "TX-total: %-"PRIu64"\n",
1612                                stats.opackets, ports_stats[pt_id].tx_dropped,
1613                                stats.opackets + ports_stats[pt_id].tx_dropped);
1614                 } else {
1615                         printf("  RX-packets:             %14"PRIu64
1616                                "    RX-dropped:%14"PRIu64
1617                                "    RX-total:%14"PRIu64"\n",
1618                                stats.ipackets, stats.imissed,
1619                                stats.ipackets + stats.imissed);
1620
1621                         if (cur_fwd_eng == &csum_fwd_engine)
1622                                 printf("  Bad-ipcsum:%14"PRIu64
1623                                        "    Bad-l4csum:%14"PRIu64
1624                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1625                                        ports_stats[pt_id].rx_bad_ip_csum,
1626                                        ports_stats[pt_id].rx_bad_l4_csum,
1627                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1628                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1629                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1630                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1631                                        stats.rx_nombuf);
1632                         }
1633
1634                         printf("  TX-packets:             %14"PRIu64
1635                                "    TX-dropped:%14"PRIu64
1636                                "    TX-total:%14"PRIu64"\n",
1637                                stats.opackets, ports_stats[pt_id].tx_dropped,
1638                                stats.opackets + ports_stats[pt_id].tx_dropped);
1639                 }
1640
1641 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1642                 if (ports_stats[pt_id].rx_stream)
1643                         pkt_burst_stats_display("RX",
1644                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1645                 if (ports_stats[pt_id].tx_stream)
1646                         pkt_burst_stats_display("TX",
1647                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1648 #endif
1649
1650                 if (port->rx_queue_stats_mapping_enabled) {
1651                         printf("\n");
1652                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1653                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1654                                        "     RX-errors:%14"PRIu64
1655                                        "    RX-bytes:%14"PRIu64"\n",
1656                                        j, stats.q_ipackets[j],
1657                                        stats.q_errors[j], stats.q_ibytes[j]);
1658                         }
1659                         printf("\n");
1660                 }
1661                 if (port->tx_queue_stats_mapping_enabled) {
1662                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1663                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1664                                        "                                 TX-bytes:%14"
1665                                        PRIu64"\n",
1666                                        j, stats.q_opackets[j],
1667                                        stats.q_obytes[j]);
1668                         }
1669                 }
1670
1671                 printf("  %s--------------------------------%s\n",
1672                        fwd_stats_border, fwd_stats_border);
1673         }
1674
1675         printf("\n  %s Accumulated forward statistics for all ports"
1676                "%s\n",
1677                acc_stats_border, acc_stats_border);
1678         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1679                "%-"PRIu64"\n"
1680                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1681                "%-"PRIu64"\n",
1682                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1683                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1684         if (total_rx_nombuf > 0)
1685                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1686         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1687                "%s\n",
1688                acc_stats_border, acc_stats_border);
1689 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1690         if (total_recv > 0)
1691                 printf("\n  CPU cycles/packet=%u (total cycles="
1692                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1693                        (unsigned int)(fwd_cycles / total_recv),
1694                        fwd_cycles, total_recv);
1695 #endif
1696 }
1697
1698 void
1699 fwd_stats_reset(void)
1700 {
1701         streamid_t sm_id;
1702         portid_t pt_id;
1703         int i;
1704
1705         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1706                 pt_id = fwd_ports_ids[i];
1707                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1708         }
1709         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1710                 struct fwd_stream *fs = fwd_streams[sm_id];
1711
1712                 fs->rx_packets = 0;
1713                 fs->tx_packets = 0;
1714                 fs->fwd_dropped = 0;
1715                 fs->rx_bad_ip_csum = 0;
1716                 fs->rx_bad_l4_csum = 0;
1717                 fs->rx_bad_outer_l4_csum = 0;
1718
1719 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1720                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1721                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1722 #endif
1723 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1724                 fs->core_cycles = 0;
1725 #endif
1726         }
1727 }
1728
1729 static void
1730 flush_fwd_rx_queues(void)
1731 {
1732         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1733         portid_t  rxp;
1734         portid_t port_id;
1735         queueid_t rxq;
1736         uint16_t  nb_rx;
1737         uint16_t  i;
1738         uint8_t   j;
1739         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1740         uint64_t timer_period;
1741
1742         /* convert to number of cycles */
1743         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1744
1745         for (j = 0; j < 2; j++) {
1746                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1747                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1748                                 port_id = fwd_ports_ids[rxp];
1749                                 /**
1750                                 * testpmd can stuck in the below do while loop
1751                                 * if rte_eth_rx_burst() always returns nonzero
1752                                 * packets. So timer is added to exit this loop
1753                                 * after 1sec timer expiry.
1754                                 */
1755                                 prev_tsc = rte_rdtsc();
1756                                 do {
1757                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1758                                                 pkts_burst, MAX_PKT_BURST);
1759                                         for (i = 0; i < nb_rx; i++)
1760                                                 rte_pktmbuf_free(pkts_burst[i]);
1761
1762                                         cur_tsc = rte_rdtsc();
1763                                         diff_tsc = cur_tsc - prev_tsc;
1764                                         timer_tsc += diff_tsc;
1765                                 } while ((nb_rx > 0) &&
1766                                         (timer_tsc < timer_period));
1767                                 timer_tsc = 0;
1768                         }
1769                 }
1770                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1771         }
1772 }
1773
1774 static void
1775 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1776 {
1777         struct fwd_stream **fsm;
1778         streamid_t nb_fs;
1779         streamid_t sm_id;
1780 #ifdef RTE_LIBRTE_BITRATE
1781         uint64_t tics_per_1sec;
1782         uint64_t tics_datum;
1783         uint64_t tics_current;
1784         uint16_t i, cnt_ports;
1785
1786         cnt_ports = nb_ports;
1787         tics_datum = rte_rdtsc();
1788         tics_per_1sec = rte_get_timer_hz();
1789 #endif
1790         fsm = &fwd_streams[fc->stream_idx];
1791         nb_fs = fc->stream_nb;
1792         do {
1793                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1794                         (*pkt_fwd)(fsm[sm_id]);
1795 #ifdef RTE_LIBRTE_BITRATE
1796                 if (bitrate_enabled != 0 &&
1797                                 bitrate_lcore_id == rte_lcore_id()) {
1798                         tics_current = rte_rdtsc();
1799                         if (tics_current - tics_datum >= tics_per_1sec) {
1800                                 /* Periodic bitrate calculation */
1801                                 for (i = 0; i < cnt_ports; i++)
1802                                         rte_stats_bitrate_calc(bitrate_data,
1803                                                 ports_ids[i]);
1804                                 tics_datum = tics_current;
1805                         }
1806                 }
1807 #endif
1808 #ifdef RTE_LIBRTE_LATENCY_STATS
1809                 if (latencystats_enabled != 0 &&
1810                                 latencystats_lcore_id == rte_lcore_id())
1811                         rte_latencystats_update();
1812 #endif
1813
1814         } while (! fc->stopped);
1815 }
1816
1817 static int
1818 start_pkt_forward_on_core(void *fwd_arg)
1819 {
1820         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1821                              cur_fwd_config.fwd_eng->packet_fwd);
1822         return 0;
1823 }
1824
1825 /*
1826  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1827  * Used to start communication flows in network loopback test configurations.
1828  */
1829 static int
1830 run_one_txonly_burst_on_core(void *fwd_arg)
1831 {
1832         struct fwd_lcore *fwd_lc;
1833         struct fwd_lcore tmp_lcore;
1834
1835         fwd_lc = (struct fwd_lcore *) fwd_arg;
1836         tmp_lcore = *fwd_lc;
1837         tmp_lcore.stopped = 1;
1838         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1839         return 0;
1840 }
1841
1842 /*
1843  * Launch packet forwarding:
1844  *     - Setup per-port forwarding context.
1845  *     - launch logical cores with their forwarding configuration.
1846  */
1847 static void
1848 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1849 {
1850         port_fwd_begin_t port_fwd_begin;
1851         unsigned int i;
1852         unsigned int lc_id;
1853         int diag;
1854
1855         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1856         if (port_fwd_begin != NULL) {
1857                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1858                         (*port_fwd_begin)(fwd_ports_ids[i]);
1859         }
1860         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1861                 lc_id = fwd_lcores_cpuids[i];
1862                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1863                         fwd_lcores[i]->stopped = 0;
1864                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1865                                                      fwd_lcores[i], lc_id);
1866                         if (diag != 0)
1867                                 printf("launch lcore %u failed - diag=%d\n",
1868                                        lc_id, diag);
1869                 }
1870         }
1871 }
1872
1873 /*
1874  * Launch packet forwarding configuration.
1875  */
1876 void
1877 start_packet_forwarding(int with_tx_first)
1878 {
1879         port_fwd_begin_t port_fwd_begin;
1880         port_fwd_end_t  port_fwd_end;
1881         struct rte_port *port;
1882         unsigned int i;
1883         portid_t   pt_id;
1884
1885         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1886                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1887
1888         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1889                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1890
1891         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1892                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1893                 (!nb_rxq || !nb_txq))
1894                 rte_exit(EXIT_FAILURE,
1895                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1896                         cur_fwd_eng->fwd_mode_name);
1897
1898         if (all_ports_started() == 0) {
1899                 printf("Not all ports were started\n");
1900                 return;
1901         }
1902         if (test_done == 0) {
1903                 printf("Packet forwarding already started\n");
1904                 return;
1905         }
1906
1907
1908         if(dcb_test) {
1909                 for (i = 0; i < nb_fwd_ports; i++) {
1910                         pt_id = fwd_ports_ids[i];
1911                         port = &ports[pt_id];
1912                         if (!port->dcb_flag) {
1913                                 printf("In DCB mode, all forwarding ports must "
1914                                        "be configured in this mode.\n");
1915                                 return;
1916                         }
1917                 }
1918                 if (nb_fwd_lcores == 1) {
1919                         printf("In DCB mode,the nb forwarding cores "
1920                                "should be larger than 1.\n");
1921                         return;
1922                 }
1923         }
1924         test_done = 0;
1925
1926         fwd_config_setup();
1927
1928         if(!no_flush_rx)
1929                 flush_fwd_rx_queues();
1930
1931         pkt_fwd_config_display(&cur_fwd_config);
1932         rxtx_config_display();
1933
1934         fwd_stats_reset();
1935         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1936                 pt_id = fwd_ports_ids[i];
1937                 port = &ports[pt_id];
1938                 map_port_queue_stats_mapping_registers(pt_id, port);
1939         }
1940         if (with_tx_first) {
1941                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1942                 if (port_fwd_begin != NULL) {
1943                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1944                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1945                 }
1946                 while (with_tx_first--) {
1947                         launch_packet_forwarding(
1948                                         run_one_txonly_burst_on_core);
1949                         rte_eal_mp_wait_lcore();
1950                 }
1951                 port_fwd_end = tx_only_engine.port_fwd_end;
1952                 if (port_fwd_end != NULL) {
1953                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1954                                 (*port_fwd_end)(fwd_ports_ids[i]);
1955                 }
1956         }
1957         launch_packet_forwarding(start_pkt_forward_on_core);
1958 }
1959
1960 void
1961 stop_packet_forwarding(void)
1962 {
1963         port_fwd_end_t port_fwd_end;
1964         lcoreid_t lc_id;
1965         portid_t pt_id;
1966         int i;
1967
1968         if (test_done) {
1969                 printf("Packet forwarding not started\n");
1970                 return;
1971         }
1972         printf("Telling cores to stop...");
1973         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1974                 fwd_lcores[lc_id]->stopped = 1;
1975         printf("\nWaiting for lcores to finish...\n");
1976         rte_eal_mp_wait_lcore();
1977         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1978         if (port_fwd_end != NULL) {
1979                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1980                         pt_id = fwd_ports_ids[i];
1981                         (*port_fwd_end)(pt_id);
1982                 }
1983         }
1984
1985         fwd_stats_display();
1986
1987         printf("\nDone.\n");
1988         test_done = 1;
1989 }
1990
1991 void
1992 dev_set_link_up(portid_t pid)
1993 {
1994         if (rte_eth_dev_set_link_up(pid) < 0)
1995                 printf("\nSet link up fail.\n");
1996 }
1997
1998 void
1999 dev_set_link_down(portid_t pid)
2000 {
2001         if (rte_eth_dev_set_link_down(pid) < 0)
2002                 printf("\nSet link down fail.\n");
2003 }
2004
2005 static int
2006 all_ports_started(void)
2007 {
2008         portid_t pi;
2009         struct rte_port *port;
2010
2011         RTE_ETH_FOREACH_DEV(pi) {
2012                 port = &ports[pi];
2013                 /* Check if there is a port which is not started */
2014                 if ((port->port_status != RTE_PORT_STARTED) &&
2015                         (port->slave_flag == 0))
2016                         return 0;
2017         }
2018
2019         /* No port is not started */
2020         return 1;
2021 }
2022
2023 int
2024 port_is_stopped(portid_t port_id)
2025 {
2026         struct rte_port *port = &ports[port_id];
2027
2028         if ((port->port_status != RTE_PORT_STOPPED) &&
2029             (port->slave_flag == 0))
2030                 return 0;
2031         return 1;
2032 }
2033
2034 int
2035 all_ports_stopped(void)
2036 {
2037         portid_t pi;
2038
2039         RTE_ETH_FOREACH_DEV(pi) {
2040                 if (!port_is_stopped(pi))
2041                         return 0;
2042         }
2043
2044         return 1;
2045 }
2046
2047 int
2048 port_is_started(portid_t port_id)
2049 {
2050         if (port_id_is_invalid(port_id, ENABLED_WARN))
2051                 return 0;
2052
2053         if (ports[port_id].port_status != RTE_PORT_STARTED)
2054                 return 0;
2055
2056         return 1;
2057 }
2058
2059 int
2060 start_port(portid_t pid)
2061 {
2062         int diag, need_check_link_status = -1;
2063         portid_t pi;
2064         queueid_t qi;
2065         struct rte_port *port;
2066         struct rte_ether_addr mac_addr;
2067
2068         if (port_id_is_invalid(pid, ENABLED_WARN))
2069                 return 0;
2070
2071         if(dcb_config)
2072                 dcb_test = 1;
2073         RTE_ETH_FOREACH_DEV(pi) {
2074                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2075                         continue;
2076
2077                 need_check_link_status = 0;
2078                 port = &ports[pi];
2079                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2080                                                  RTE_PORT_HANDLING) == 0) {
2081                         printf("Port %d is now not stopped\n", pi);
2082                         continue;
2083                 }
2084
2085                 if (port->need_reconfig > 0) {
2086                         port->need_reconfig = 0;
2087
2088                         if (flow_isolate_all) {
2089                                 int ret = port_flow_isolate(pi, 1);
2090                                 if (ret) {
2091                                         printf("Failed to apply isolated"
2092                                                " mode on port %d\n", pi);
2093                                         return -1;
2094                                 }
2095                         }
2096                         configure_rxtx_dump_callbacks(0);
2097                         printf("Configuring Port %d (socket %u)\n", pi,
2098                                         port->socket_id);
2099                         /* configure port */
2100                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
2101                                                 &(port->dev_conf));
2102                         if (diag != 0) {
2103                                 if (rte_atomic16_cmpset(&(port->port_status),
2104                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2105                                         printf("Port %d can not be set back "
2106                                                         "to stopped\n", pi);
2107                                 printf("Fail to configure port %d\n", pi);
2108                                 /* try to reconfigure port next time */
2109                                 port->need_reconfig = 1;
2110                                 return -1;
2111                         }
2112                 }
2113                 if (port->need_reconfig_queues > 0) {
2114                         port->need_reconfig_queues = 0;
2115                         /* setup tx queues */
2116                         for (qi = 0; qi < nb_txq; qi++) {
2117                                 if ((numa_support) &&
2118                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2119                                         diag = rte_eth_tx_queue_setup(pi, qi,
2120                                                 port->nb_tx_desc[qi],
2121                                                 txring_numa[pi],
2122                                                 &(port->tx_conf[qi]));
2123                                 else
2124                                         diag = rte_eth_tx_queue_setup(pi, qi,
2125                                                 port->nb_tx_desc[qi],
2126                                                 port->socket_id,
2127                                                 &(port->tx_conf[qi]));
2128
2129                                 if (diag == 0)
2130                                         continue;
2131
2132                                 /* Fail to setup tx queue, return */
2133                                 if (rte_atomic16_cmpset(&(port->port_status),
2134                                                         RTE_PORT_HANDLING,
2135                                                         RTE_PORT_STOPPED) == 0)
2136                                         printf("Port %d can not be set back "
2137                                                         "to stopped\n", pi);
2138                                 printf("Fail to configure port %d tx queues\n",
2139                                        pi);
2140                                 /* try to reconfigure queues next time */
2141                                 port->need_reconfig_queues = 1;
2142                                 return -1;
2143                         }
2144                         for (qi = 0; qi < nb_rxq; qi++) {
2145                                 /* setup rx queues */
2146                                 if ((numa_support) &&
2147                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2148                                         struct rte_mempool * mp =
2149                                                 mbuf_pool_find(rxring_numa[pi]);
2150                                         if (mp == NULL) {
2151                                                 printf("Failed to setup RX queue:"
2152                                                         "No mempool allocation"
2153                                                         " on the socket %d\n",
2154                                                         rxring_numa[pi]);
2155                                                 return -1;
2156                                         }
2157
2158                                         diag = rte_eth_rx_queue_setup(pi, qi,
2159                                              port->nb_rx_desc[qi],
2160                                              rxring_numa[pi],
2161                                              &(port->rx_conf[qi]),
2162                                              mp);
2163                                 } else {
2164                                         struct rte_mempool *mp =
2165                                                 mbuf_pool_find(port->socket_id);
2166                                         if (mp == NULL) {
2167                                                 printf("Failed to setup RX queue:"
2168                                                         "No mempool allocation"
2169                                                         " on the socket %d\n",
2170                                                         port->socket_id);
2171                                                 return -1;
2172                                         }
2173                                         diag = rte_eth_rx_queue_setup(pi, qi,
2174                                              port->nb_rx_desc[qi],
2175                                              port->socket_id,
2176                                              &(port->rx_conf[qi]),
2177                                              mp);
2178                                 }
2179                                 if (diag == 0)
2180                                         continue;
2181
2182                                 /* Fail to setup rx queue, return */
2183                                 if (rte_atomic16_cmpset(&(port->port_status),
2184                                                         RTE_PORT_HANDLING,
2185                                                         RTE_PORT_STOPPED) == 0)
2186                                         printf("Port %d can not be set back "
2187                                                         "to stopped\n", pi);
2188                                 printf("Fail to configure port %d rx queues\n",
2189                                        pi);
2190                                 /* try to reconfigure queues next time */
2191                                 port->need_reconfig_queues = 1;
2192                                 return -1;
2193                         }
2194                 }
2195                 configure_rxtx_dump_callbacks(verbose_level);
2196                 /* start port */
2197                 if (rte_eth_dev_start(pi) < 0) {
2198                         printf("Fail to start port %d\n", pi);
2199
2200                         /* Fail to setup rx queue, return */
2201                         if (rte_atomic16_cmpset(&(port->port_status),
2202                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2203                                 printf("Port %d can not be set back to "
2204                                                         "stopped\n", pi);
2205                         continue;
2206                 }
2207
2208                 if (rte_atomic16_cmpset(&(port->port_status),
2209                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2210                         printf("Port %d can not be set into started\n", pi);
2211
2212                 rte_eth_macaddr_get(pi, &mac_addr);
2213                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2214                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2215                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2216                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2217
2218                 /* at least one port started, need checking link status */
2219                 need_check_link_status = 1;
2220         }
2221
2222         if (need_check_link_status == 1 && !no_link_check)
2223                 check_all_ports_link_status(RTE_PORT_ALL);
2224         else if (need_check_link_status == 0)
2225                 printf("Please stop the ports first\n");
2226
2227         printf("Done\n");
2228         return 0;
2229 }
2230
2231 void
2232 stop_port(portid_t pid)
2233 {
2234         portid_t pi;
2235         struct rte_port *port;
2236         int need_check_link_status = 0;
2237
2238         if (dcb_test) {
2239                 dcb_test = 0;
2240                 dcb_config = 0;
2241         }
2242
2243         if (port_id_is_invalid(pid, ENABLED_WARN))
2244                 return;
2245
2246         printf("Stopping ports...\n");
2247
2248         RTE_ETH_FOREACH_DEV(pi) {
2249                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2250                         continue;
2251
2252                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2253                         printf("Please remove port %d from forwarding configuration.\n", pi);
2254                         continue;
2255                 }
2256
2257                 if (port_is_bonding_slave(pi)) {
2258                         printf("Please remove port %d from bonded device.\n", pi);
2259                         continue;
2260                 }
2261
2262                 port = &ports[pi];
2263                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2264                                                 RTE_PORT_HANDLING) == 0)
2265                         continue;
2266
2267                 rte_eth_dev_stop(pi);
2268
2269                 if (rte_atomic16_cmpset(&(port->port_status),
2270                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2271                         printf("Port %d can not be set into stopped\n", pi);
2272                 need_check_link_status = 1;
2273         }
2274         if (need_check_link_status && !no_link_check)
2275                 check_all_ports_link_status(RTE_PORT_ALL);
2276
2277         printf("Done\n");
2278 }
2279
2280 static void
2281 remove_invalid_ports_in(portid_t *array, portid_t *total)
2282 {
2283         portid_t i;
2284         portid_t new_total = 0;
2285
2286         for (i = 0; i < *total; i++)
2287                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2288                         array[new_total] = array[i];
2289                         new_total++;
2290                 }
2291         *total = new_total;
2292 }
2293
2294 static void
2295 remove_invalid_ports(void)
2296 {
2297         remove_invalid_ports_in(ports_ids, &nb_ports);
2298         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2299         nb_cfg_ports = nb_fwd_ports;
2300 }
2301
2302 void
2303 close_port(portid_t pid)
2304 {
2305         portid_t pi;
2306         struct rte_port *port;
2307
2308         if (port_id_is_invalid(pid, ENABLED_WARN))
2309                 return;
2310
2311         printf("Closing ports...\n");
2312
2313         RTE_ETH_FOREACH_DEV(pi) {
2314                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2315                         continue;
2316
2317                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2318                         printf("Please remove port %d from forwarding configuration.\n", pi);
2319                         continue;
2320                 }
2321
2322                 if (port_is_bonding_slave(pi)) {
2323                         printf("Please remove port %d from bonded device.\n", pi);
2324                         continue;
2325                 }
2326
2327                 port = &ports[pi];
2328                 if (rte_atomic16_cmpset(&(port->port_status),
2329                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2330                         printf("Port %d is already closed\n", pi);
2331                         continue;
2332                 }
2333
2334                 if (rte_atomic16_cmpset(&(port->port_status),
2335                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2336                         printf("Port %d is now not stopped\n", pi);
2337                         continue;
2338                 }
2339
2340                 if (port->flow_list)
2341                         port_flow_flush(pi);
2342                 rte_eth_dev_close(pi);
2343
2344                 remove_invalid_ports();
2345
2346                 if (rte_atomic16_cmpset(&(port->port_status),
2347                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2348                         printf("Port %d cannot be set to closed\n", pi);
2349         }
2350
2351         printf("Done\n");
2352 }
2353
2354 void
2355 reset_port(portid_t pid)
2356 {
2357         int diag;
2358         portid_t pi;
2359         struct rte_port *port;
2360
2361         if (port_id_is_invalid(pid, ENABLED_WARN))
2362                 return;
2363
2364         printf("Resetting ports...\n");
2365
2366         RTE_ETH_FOREACH_DEV(pi) {
2367                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2368                         continue;
2369
2370                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2371                         printf("Please remove port %d from forwarding "
2372                                "configuration.\n", pi);
2373                         continue;
2374                 }
2375
2376                 if (port_is_bonding_slave(pi)) {
2377                         printf("Please remove port %d from bonded device.\n",
2378                                pi);
2379                         continue;
2380                 }
2381
2382                 diag = rte_eth_dev_reset(pi);
2383                 if (diag == 0) {
2384                         port = &ports[pi];
2385                         port->need_reconfig = 1;
2386                         port->need_reconfig_queues = 1;
2387                 } else {
2388                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2389                 }
2390         }
2391
2392         printf("Done\n");
2393 }
2394
2395 void
2396 attach_port(char *identifier)
2397 {
2398         portid_t pi;
2399         struct rte_dev_iterator iterator;
2400
2401         printf("Attaching a new port...\n");
2402
2403         if (identifier == NULL) {
2404                 printf("Invalid parameters are specified\n");
2405                 return;
2406         }
2407
2408         if (rte_dev_probe(identifier) < 0) {
2409                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2410                 return;
2411         }
2412
2413         /* first attach mode: event */
2414         if (setup_on_probe_event) {
2415                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2416                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2417                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2418                                         ports[pi].need_setup != 0)
2419                                 setup_attached_port(pi);
2420                 return;
2421         }
2422
2423         /* second attach mode: iterator */
2424         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2425                 /* setup ports matching the devargs used for probing */
2426                 if (port_is_forwarding(pi))
2427                         continue; /* port was already attached before */
2428                 setup_attached_port(pi);
2429         }
2430 }
2431
2432 static void
2433 setup_attached_port(portid_t pi)
2434 {
2435         unsigned int socket_id;
2436
2437         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2438         /* if socket_id is invalid, set to the first available socket. */
2439         if (check_socket_id(socket_id) < 0)
2440                 socket_id = socket_ids[0];
2441         reconfig(pi, socket_id);
2442         rte_eth_promiscuous_enable(pi);
2443
2444         ports_ids[nb_ports++] = pi;
2445         fwd_ports_ids[nb_fwd_ports++] = pi;
2446         nb_cfg_ports = nb_fwd_ports;
2447         ports[pi].need_setup = 0;
2448         ports[pi].port_status = RTE_PORT_STOPPED;
2449
2450         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2451         printf("Done\n");
2452 }
2453
2454 void
2455 detach_port_device(portid_t port_id)
2456 {
2457         struct rte_device *dev;
2458         portid_t sibling;
2459
2460         printf("Removing a device...\n");
2461
2462         dev = rte_eth_devices[port_id].device;
2463         if (dev == NULL) {
2464                 printf("Device already removed\n");
2465                 return;
2466         }
2467
2468         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2469                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2470                         printf("Port not stopped\n");
2471                         return;
2472                 }
2473                 printf("Port was not closed\n");
2474                 if (ports[port_id].flow_list)
2475                         port_flow_flush(port_id);
2476         }
2477
2478         if (rte_dev_remove(dev) < 0) {
2479                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2480                 return;
2481         }
2482         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2483                 /* reset mapping between old ports and removed device */
2484                 rte_eth_devices[sibling].device = NULL;
2485                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2486                         /* sibling ports are forced to be closed */
2487                         ports[sibling].port_status = RTE_PORT_CLOSED;
2488                         printf("Port %u is closed\n", sibling);
2489                 }
2490         }
2491
2492         remove_invalid_ports();
2493
2494         printf("Device of port %u is detached\n", port_id);
2495         printf("Now total ports is %d\n", nb_ports);
2496         printf("Done\n");
2497         return;
2498 }
2499
2500 void
2501 detach_device(char *identifier)
2502 {
2503         struct rte_dev_iterator iterator;
2504         struct rte_devargs da;
2505         portid_t port_id;
2506
2507         printf("Removing a device...\n");
2508
2509         memset(&da, 0, sizeof(da));
2510         if (rte_devargs_parsef(&da, "%s", identifier)) {
2511                 printf("cannot parse identifier\n");
2512                 if (da.args)
2513                         free(da.args);
2514                 return;
2515         }
2516
2517         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2518                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2519                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2520                                 printf("Port %u not stopped\n", port_id);
2521                                 return;
2522                         }
2523
2524                         /* sibling ports are forced to be closed */
2525                         if (ports[port_id].flow_list)
2526                                 port_flow_flush(port_id);
2527                         ports[port_id].port_status = RTE_PORT_CLOSED;
2528                         printf("Port %u is now closed\n", port_id);
2529                 }
2530         }
2531
2532         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2533                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2534                             da.name, da.bus->name);
2535                 return;
2536         }
2537
2538         remove_invalid_ports();
2539
2540         printf("Device %s is detached\n", identifier);
2541         printf("Now total ports is %d\n", nb_ports);
2542         printf("Done\n");
2543 }
2544
2545 void
2546 pmd_test_exit(void)
2547 {
2548         portid_t pt_id;
2549         int ret;
2550         int i;
2551
2552         if (test_done == 0)
2553                 stop_packet_forwarding();
2554
2555         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2556                 if (mempools[i]) {
2557                         if (mp_alloc_type == MP_ALLOC_ANON)
2558                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2559                                                      NULL);
2560                 }
2561         }
2562         if (ports != NULL) {
2563                 no_link_check = 1;
2564                 RTE_ETH_FOREACH_DEV(pt_id) {
2565                         printf("\nStopping port %d...\n", pt_id);
2566                         fflush(stdout);
2567                         stop_port(pt_id);
2568                 }
2569                 RTE_ETH_FOREACH_DEV(pt_id) {
2570                         printf("\nShutting down port %d...\n", pt_id);
2571                         fflush(stdout);
2572                         close_port(pt_id);
2573                 }
2574         }
2575
2576         if (hot_plug) {
2577                 ret = rte_dev_event_monitor_stop();
2578                 if (ret) {
2579                         RTE_LOG(ERR, EAL,
2580                                 "fail to stop device event monitor.");
2581                         return;
2582                 }
2583
2584                 ret = rte_dev_event_callback_unregister(NULL,
2585                         dev_event_callback, NULL);
2586                 if (ret < 0) {
2587                         RTE_LOG(ERR, EAL,
2588                                 "fail to unregister device event callback.\n");
2589                         return;
2590                 }
2591
2592                 ret = rte_dev_hotplug_handle_disable();
2593                 if (ret) {
2594                         RTE_LOG(ERR, EAL,
2595                                 "fail to disable hotplug handling.\n");
2596                         return;
2597                 }
2598         }
2599         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2600                 if (mempools[i])
2601                         rte_mempool_free(mempools[i]);
2602         }
2603
2604         printf("\nBye...\n");
2605 }
2606
2607 typedef void (*cmd_func_t)(void);
2608 struct pmd_test_command {
2609         const char *cmd_name;
2610         cmd_func_t cmd_func;
2611 };
2612
2613 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2614
2615 /* Check the link status of all ports in up to 9s, and print them finally */
2616 static void
2617 check_all_ports_link_status(uint32_t port_mask)
2618 {
2619 #define CHECK_INTERVAL 100 /* 100ms */
2620 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2621         portid_t portid;
2622         uint8_t count, all_ports_up, print_flag = 0;
2623         struct rte_eth_link link;
2624
2625         printf("Checking link statuses...\n");
2626         fflush(stdout);
2627         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2628                 all_ports_up = 1;
2629                 RTE_ETH_FOREACH_DEV(portid) {
2630                         if ((port_mask & (1 << portid)) == 0)
2631                                 continue;
2632                         memset(&link, 0, sizeof(link));
2633                         rte_eth_link_get_nowait(portid, &link);
2634                         /* print link status if flag set */
2635                         if (print_flag == 1) {
2636                                 if (link.link_status)
2637                                         printf(
2638                                         "Port%d Link Up. speed %u Mbps- %s\n",
2639                                         portid, link.link_speed,
2640                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2641                                         ("full-duplex") : ("half-duplex\n"));
2642                                 else
2643                                         printf("Port %d Link Down\n", portid);
2644                                 continue;
2645                         }
2646                         /* clear all_ports_up flag if any link down */
2647                         if (link.link_status == ETH_LINK_DOWN) {
2648                                 all_ports_up = 0;
2649                                 break;
2650                         }
2651                 }
2652                 /* after finally printing all link status, get out */
2653                 if (print_flag == 1)
2654                         break;
2655
2656                 if (all_ports_up == 0) {
2657                         fflush(stdout);
2658                         rte_delay_ms(CHECK_INTERVAL);
2659                 }
2660
2661                 /* set the print_flag if all ports up or timeout */
2662                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2663                         print_flag = 1;
2664                 }
2665
2666                 if (lsc_interrupt)
2667                         break;
2668         }
2669 }
2670
2671 /*
2672  * This callback is for remove a port for a device. It has limitation because
2673  * it is not for multiple port removal for a device.
2674  * TODO: the device detach invoke will plan to be removed from user side to
2675  * eal. And convert all PMDs to free port resources on ether device closing.
2676  */
2677 static void
2678 rmv_port_callback(void *arg)
2679 {
2680         int need_to_start = 0;
2681         int org_no_link_check = no_link_check;
2682         portid_t port_id = (intptr_t)arg;
2683
2684         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2685
2686         if (!test_done && port_is_forwarding(port_id)) {
2687                 need_to_start = 1;
2688                 stop_packet_forwarding();
2689         }
2690         no_link_check = 1;
2691         stop_port(port_id);
2692         no_link_check = org_no_link_check;
2693         close_port(port_id);
2694         detach_port_device(port_id);
2695         if (need_to_start)
2696                 start_packet_forwarding(0);
2697 }
2698
2699 /* This function is used by the interrupt thread */
2700 static int
2701 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2702                   void *ret_param)
2703 {
2704         RTE_SET_USED(param);
2705         RTE_SET_USED(ret_param);
2706
2707         if (type >= RTE_ETH_EVENT_MAX) {
2708                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2709                         port_id, __func__, type);
2710                 fflush(stderr);
2711         } else if (event_print_mask & (UINT32_C(1) << type)) {
2712                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2713                         eth_event_desc[type]);
2714                 fflush(stdout);
2715         }
2716
2717         switch (type) {
2718         case RTE_ETH_EVENT_NEW:
2719                 ports[port_id].need_setup = 1;
2720                 ports[port_id].port_status = RTE_PORT_HANDLING;
2721                 break;
2722         case RTE_ETH_EVENT_INTR_RMV:
2723                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2724                         break;
2725                 if (rte_eal_alarm_set(100000,
2726                                 rmv_port_callback, (void *)(intptr_t)port_id))
2727                         fprintf(stderr, "Could not set up deferred device removal\n");
2728                 break;
2729         default:
2730                 break;
2731         }
2732         return 0;
2733 }
2734
2735 static int
2736 register_eth_event_callback(void)
2737 {
2738         int ret;
2739         enum rte_eth_event_type event;
2740
2741         for (event = RTE_ETH_EVENT_UNKNOWN;
2742                         event < RTE_ETH_EVENT_MAX; event++) {
2743                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2744                                 event,
2745                                 eth_event_callback,
2746                                 NULL);
2747                 if (ret != 0) {
2748                         TESTPMD_LOG(ERR, "Failed to register callback for "
2749                                         "%s event\n", eth_event_desc[event]);
2750                         return -1;
2751                 }
2752         }
2753
2754         return 0;
2755 }
2756
2757 /* This function is used by the interrupt thread */
2758 static void
2759 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2760                              __rte_unused void *arg)
2761 {
2762         uint16_t port_id;
2763         int ret;
2764
2765         if (type >= RTE_DEV_EVENT_MAX) {
2766                 fprintf(stderr, "%s called upon invalid event %d\n",
2767                         __func__, type);
2768                 fflush(stderr);
2769         }
2770
2771         switch (type) {
2772         case RTE_DEV_EVENT_REMOVE:
2773                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2774                         device_name);
2775                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2776                 if (ret) {
2777                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2778                                 device_name);
2779                         return;
2780                 }
2781                 /*
2782                  * Because the user's callback is invoked in eal interrupt
2783                  * callback, the interrupt callback need to be finished before
2784                  * it can be unregistered when detaching device. So finish
2785                  * callback soon and use a deferred removal to detach device
2786                  * is need. It is a workaround, once the device detaching be
2787                  * moved into the eal in the future, the deferred removal could
2788                  * be deleted.
2789                  */
2790                 if (rte_eal_alarm_set(100000,
2791                                 rmv_port_callback, (void *)(intptr_t)port_id))
2792                         RTE_LOG(ERR, EAL,
2793                                 "Could not set up deferred device removal\n");
2794                 break;
2795         case RTE_DEV_EVENT_ADD:
2796                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2797                         device_name);
2798                 /* TODO: After finish kernel driver binding,
2799                  * begin to attach port.
2800                  */
2801                 break;
2802         default:
2803                 break;
2804         }
2805 }
2806
2807 static int
2808 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2809 {
2810         uint16_t i;
2811         int diag;
2812         uint8_t mapping_found = 0;
2813
2814         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2815                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2816                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2817                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2818                                         tx_queue_stats_mappings[i].queue_id,
2819                                         tx_queue_stats_mappings[i].stats_counter_id);
2820                         if (diag != 0)
2821                                 return diag;
2822                         mapping_found = 1;
2823                 }
2824         }
2825         if (mapping_found)
2826                 port->tx_queue_stats_mapping_enabled = 1;
2827         return 0;
2828 }
2829
2830 static int
2831 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2832 {
2833         uint16_t i;
2834         int diag;
2835         uint8_t mapping_found = 0;
2836
2837         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2838                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2839                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2840                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2841                                         rx_queue_stats_mappings[i].queue_id,
2842                                         rx_queue_stats_mappings[i].stats_counter_id);
2843                         if (diag != 0)
2844                                 return diag;
2845                         mapping_found = 1;
2846                 }
2847         }
2848         if (mapping_found)
2849                 port->rx_queue_stats_mapping_enabled = 1;
2850         return 0;
2851 }
2852
2853 static void
2854 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2855 {
2856         int diag = 0;
2857
2858         diag = set_tx_queue_stats_mapping_registers(pi, port);
2859         if (diag != 0) {
2860                 if (diag == -ENOTSUP) {
2861                         port->tx_queue_stats_mapping_enabled = 0;
2862                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2863                 }
2864                 else
2865                         rte_exit(EXIT_FAILURE,
2866                                         "set_tx_queue_stats_mapping_registers "
2867                                         "failed for port id=%d diag=%d\n",
2868                                         pi, diag);
2869         }
2870
2871         diag = set_rx_queue_stats_mapping_registers(pi, port);
2872         if (diag != 0) {
2873                 if (diag == -ENOTSUP) {
2874                         port->rx_queue_stats_mapping_enabled = 0;
2875                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2876                 }
2877                 else
2878                         rte_exit(EXIT_FAILURE,
2879                                         "set_rx_queue_stats_mapping_registers "
2880                                         "failed for port id=%d diag=%d\n",
2881                                         pi, diag);
2882         }
2883 }
2884
2885 static void
2886 rxtx_port_config(struct rte_port *port)
2887 {
2888         uint16_t qid;
2889         uint64_t offloads;
2890
2891         for (qid = 0; qid < nb_rxq; qid++) {
2892                 offloads = port->rx_conf[qid].offloads;
2893                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2894                 if (offloads != 0)
2895                         port->rx_conf[qid].offloads = offloads;
2896
2897                 /* Check if any Rx parameters have been passed */
2898                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2899                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2900
2901                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2902                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2903
2904                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2905                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2906
2907                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2908                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2909
2910                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2911                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2912
2913                 port->nb_rx_desc[qid] = nb_rxd;
2914         }
2915
2916         for (qid = 0; qid < nb_txq; qid++) {
2917                 offloads = port->tx_conf[qid].offloads;
2918                 port->tx_conf[qid] = port->dev_info.default_txconf;
2919                 if (offloads != 0)
2920                         port->tx_conf[qid].offloads = offloads;
2921
2922                 /* Check if any Tx parameters have been passed */
2923                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2924                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2925
2926                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2927                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2928
2929                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2930                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2931
2932                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2933                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2934
2935                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2936                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2937
2938                 port->nb_tx_desc[qid] = nb_txd;
2939         }
2940 }
2941
2942 void
2943 init_port_config(void)
2944 {
2945         portid_t pid;
2946         struct rte_port *port;
2947         int ret;
2948
2949         RTE_ETH_FOREACH_DEV(pid) {
2950                 port = &ports[pid];
2951                 port->dev_conf.fdir_conf = fdir_conf;
2952
2953                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
2954                 if (ret != 0)
2955                         return;
2956
2957                 if (nb_rxq > 1) {
2958                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2959                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2960                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2961                 } else {
2962                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2963                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2964                 }
2965
2966                 if (port->dcb_flag == 0) {
2967                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2968                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2969                         else
2970                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2971                 }
2972
2973                 rxtx_port_config(port);
2974
2975                 rte_eth_macaddr_get(pid, &port->eth_addr);
2976
2977                 map_port_queue_stats_mapping_registers(pid, port);
2978 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2979                 rte_pmd_ixgbe_bypass_init(pid);
2980 #endif
2981
2982                 if (lsc_interrupt &&
2983                     (rte_eth_devices[pid].data->dev_flags &
2984                      RTE_ETH_DEV_INTR_LSC))
2985                         port->dev_conf.intr_conf.lsc = 1;
2986                 if (rmv_interrupt &&
2987                     (rte_eth_devices[pid].data->dev_flags &
2988                      RTE_ETH_DEV_INTR_RMV))
2989                         port->dev_conf.intr_conf.rmv = 1;
2990         }
2991 }
2992
2993 void set_port_slave_flag(portid_t slave_pid)
2994 {
2995         struct rte_port *port;
2996
2997         port = &ports[slave_pid];
2998         port->slave_flag = 1;
2999 }
3000
3001 void clear_port_slave_flag(portid_t slave_pid)
3002 {
3003         struct rte_port *port;
3004
3005         port = &ports[slave_pid];
3006         port->slave_flag = 0;
3007 }
3008
3009 uint8_t port_is_bonding_slave(portid_t slave_pid)
3010 {
3011         struct rte_port *port;
3012
3013         port = &ports[slave_pid];
3014         if ((rte_eth_devices[slave_pid].data->dev_flags &
3015             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3016                 return 1;
3017         return 0;
3018 }
3019
3020 const uint16_t vlan_tags[] = {
3021                 0,  1,  2,  3,  4,  5,  6,  7,
3022                 8,  9, 10, 11,  12, 13, 14, 15,
3023                 16, 17, 18, 19, 20, 21, 22, 23,
3024                 24, 25, 26, 27, 28, 29, 30, 31
3025 };
3026
3027 static  int
3028 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3029                  enum dcb_mode_enable dcb_mode,
3030                  enum rte_eth_nb_tcs num_tcs,
3031                  uint8_t pfc_en)
3032 {
3033         uint8_t i;
3034         int32_t rc;
3035         struct rte_eth_rss_conf rss_conf;
3036
3037         /*
3038          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3039          * given above, and the number of traffic classes available for use.
3040          */
3041         if (dcb_mode == DCB_VT_ENABLED) {
3042                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3043                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3044                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3045                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3046
3047                 /* VMDQ+DCB RX and TX configurations */
3048                 vmdq_rx_conf->enable_default_pool = 0;
3049                 vmdq_rx_conf->default_pool = 0;
3050                 vmdq_rx_conf->nb_queue_pools =
3051                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3052                 vmdq_tx_conf->nb_queue_pools =
3053                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3054
3055                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3056                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3057                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3058                         vmdq_rx_conf->pool_map[i].pools =
3059                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3060                 }
3061                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3062                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3063                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3064                 }
3065
3066                 /* set DCB mode of RX and TX of multiple queues */
3067                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
3068                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3069         } else {
3070                 struct rte_eth_dcb_rx_conf *rx_conf =
3071                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3072                 struct rte_eth_dcb_tx_conf *tx_conf =
3073                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3074
3075                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3076                 if (rc != 0)
3077                         return rc;
3078
3079                 rx_conf->nb_tcs = num_tcs;
3080                 tx_conf->nb_tcs = num_tcs;
3081
3082                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3083                         rx_conf->dcb_tc[i] = i % num_tcs;
3084                         tx_conf->dcb_tc[i] = i % num_tcs;
3085                 }
3086
3087                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
3088                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3089                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3090         }
3091
3092         if (pfc_en)
3093                 eth_conf->dcb_capability_en =
3094                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3095         else
3096                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3097
3098         return 0;
3099 }
3100
3101 int
3102 init_port_dcb_config(portid_t pid,
3103                      enum dcb_mode_enable dcb_mode,
3104                      enum rte_eth_nb_tcs num_tcs,
3105                      uint8_t pfc_en)
3106 {
3107         struct rte_eth_conf port_conf;
3108         struct rte_port *rte_port;
3109         int retval;
3110         uint16_t i;
3111
3112         rte_port = &ports[pid];
3113
3114         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3115         /* Enter DCB configuration status */
3116         dcb_config = 1;
3117
3118         port_conf.rxmode = rte_port->dev_conf.rxmode;
3119         port_conf.txmode = rte_port->dev_conf.txmode;
3120
3121         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3122         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3123         if (retval < 0)
3124                 return retval;
3125         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3126
3127         /* re-configure the device . */
3128         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3129         if (retval < 0)
3130                 return retval;
3131
3132         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3133         if (retval != 0)
3134                 return retval;
3135
3136         /* If dev_info.vmdq_pool_base is greater than 0,
3137          * the queue id of vmdq pools is started after pf queues.
3138          */
3139         if (dcb_mode == DCB_VT_ENABLED &&
3140             rte_port->dev_info.vmdq_pool_base > 0) {
3141                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3142                         " for port %d.", pid);
3143                 return -1;
3144         }
3145
3146         /* Assume the ports in testpmd have the same dcb capability
3147          * and has the same number of rxq and txq in dcb mode
3148          */
3149         if (dcb_mode == DCB_VT_ENABLED) {
3150                 if (rte_port->dev_info.max_vfs > 0) {
3151                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3152                         nb_txq = rte_port->dev_info.nb_tx_queues;
3153                 } else {
3154                         nb_rxq = rte_port->dev_info.max_rx_queues;
3155                         nb_txq = rte_port->dev_info.max_tx_queues;
3156                 }
3157         } else {
3158                 /*if vt is disabled, use all pf queues */
3159                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3160                         nb_rxq = rte_port->dev_info.max_rx_queues;
3161                         nb_txq = rte_port->dev_info.max_tx_queues;
3162                 } else {
3163                         nb_rxq = (queueid_t)num_tcs;
3164                         nb_txq = (queueid_t)num_tcs;
3165
3166                 }
3167         }
3168         rx_free_thresh = 64;
3169
3170         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3171
3172         rxtx_port_config(rte_port);
3173         /* VLAN filter */
3174         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3175         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3176                 rx_vft_set(pid, vlan_tags[i], 1);
3177
3178         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
3179         map_port_queue_stats_mapping_registers(pid, rte_port);
3180
3181         rte_port->dcb_flag = 1;
3182
3183         return 0;
3184 }
3185
3186 static void
3187 init_port(void)
3188 {
3189         /* Configuration of Ethernet ports. */
3190         ports = rte_zmalloc("testpmd: ports",
3191                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3192                             RTE_CACHE_LINE_SIZE);
3193         if (ports == NULL) {
3194                 rte_exit(EXIT_FAILURE,
3195                                 "rte_zmalloc(%d struct rte_port) failed\n",
3196                                 RTE_MAX_ETHPORTS);
3197         }
3198
3199         /* Initialize ports NUMA structures */
3200         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3201         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3202         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3203 }
3204
3205 static void
3206 force_quit(void)
3207 {
3208         pmd_test_exit();
3209         prompt_exit();
3210 }
3211
3212 static void
3213 print_stats(void)
3214 {
3215         uint8_t i;
3216         const char clr[] = { 27, '[', '2', 'J', '\0' };
3217         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3218
3219         /* Clear screen and move to top left */
3220         printf("%s%s", clr, top_left);
3221
3222         printf("\nPort statistics ====================================");
3223         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3224                 nic_stats_display(fwd_ports_ids[i]);
3225
3226         fflush(stdout);
3227 }
3228
3229 static void
3230 signal_handler(int signum)
3231 {
3232         if (signum == SIGINT || signum == SIGTERM) {
3233                 printf("\nSignal %d received, preparing to exit...\n",
3234                                 signum);
3235 #ifdef RTE_LIBRTE_PDUMP
3236                 /* uninitialize packet capture framework */
3237                 rte_pdump_uninit();
3238 #endif
3239 #ifdef RTE_LIBRTE_LATENCY_STATS
3240                 if (latencystats_enabled != 0)
3241                         rte_latencystats_uninit();
3242 #endif
3243                 force_quit();
3244                 /* Set flag to indicate the force termination. */
3245                 f_quit = 1;
3246                 /* exit with the expected status */
3247                 signal(signum, SIG_DFL);
3248                 kill(getpid(), signum);
3249         }
3250 }
3251
3252 int
3253 main(int argc, char** argv)
3254 {
3255         int diag;
3256         portid_t port_id;
3257         uint16_t count;
3258         int ret;
3259
3260         signal(SIGINT, signal_handler);
3261         signal(SIGTERM, signal_handler);
3262
3263         testpmd_logtype = rte_log_register("testpmd");
3264         if (testpmd_logtype < 0)
3265                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3266         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3267
3268         diag = rte_eal_init(argc, argv);
3269         if (diag < 0)
3270                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3271                          rte_strerror(rte_errno));
3272
3273         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3274                 rte_exit(EXIT_FAILURE,
3275                          "Secondary process type not supported.\n");
3276
3277         ret = register_eth_event_callback();
3278         if (ret != 0)
3279                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3280
3281 #ifdef RTE_LIBRTE_PDUMP
3282         /* initialize packet capture framework */
3283         rte_pdump_init();
3284 #endif
3285
3286         count = 0;
3287         RTE_ETH_FOREACH_DEV(port_id) {
3288                 ports_ids[count] = port_id;
3289                 count++;
3290         }
3291         nb_ports = (portid_t) count;
3292         if (nb_ports == 0)
3293                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3294
3295         /* allocate port structures, and init them */
3296         init_port();
3297
3298         set_def_fwd_config();
3299         if (nb_lcores == 0)
3300                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3301                          "Check the core mask argument\n");
3302
3303         /* Bitrate/latency stats disabled by default */
3304 #ifdef RTE_LIBRTE_BITRATE
3305         bitrate_enabled = 0;
3306 #endif
3307 #ifdef RTE_LIBRTE_LATENCY_STATS
3308         latencystats_enabled = 0;
3309 #endif
3310
3311         /* on FreeBSD, mlockall() is disabled by default */
3312 #ifdef RTE_EXEC_ENV_FREEBSD
3313         do_mlockall = 0;
3314 #else
3315         do_mlockall = 1;
3316 #endif
3317
3318         argc -= diag;
3319         argv += diag;
3320         if (argc > 1)
3321                 launch_args_parse(argc, argv);
3322
3323         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3324                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3325                         strerror(errno));
3326         }
3327
3328         if (tx_first && interactive)
3329                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3330                                 "interactive mode.\n");
3331
3332         if (tx_first && lsc_interrupt) {
3333                 printf("Warning: lsc_interrupt needs to be off when "
3334                                 " using tx_first. Disabling.\n");
3335                 lsc_interrupt = 0;
3336         }
3337
3338         if (!nb_rxq && !nb_txq)
3339                 printf("Warning: Either rx or tx queues should be non-zero\n");
3340
3341         if (nb_rxq > 1 && nb_rxq > nb_txq)
3342                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3343                        "but nb_txq=%d will prevent to fully test it.\n",
3344                        nb_rxq, nb_txq);
3345
3346         init_config();
3347
3348         if (hot_plug) {
3349                 ret = rte_dev_hotplug_handle_enable();
3350                 if (ret) {
3351                         RTE_LOG(ERR, EAL,
3352                                 "fail to enable hotplug handling.");
3353                         return -1;
3354                 }
3355
3356                 ret = rte_dev_event_monitor_start();
3357                 if (ret) {
3358                         RTE_LOG(ERR, EAL,
3359                                 "fail to start device event monitoring.");
3360                         return -1;
3361                 }
3362
3363                 ret = rte_dev_event_callback_register(NULL,
3364                         dev_event_callback, NULL);
3365                 if (ret) {
3366                         RTE_LOG(ERR, EAL,
3367                                 "fail  to register device event callback\n");
3368                         return -1;
3369                 }
3370         }
3371
3372         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3373                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3374
3375         /* set all ports to promiscuous mode by default */
3376         RTE_ETH_FOREACH_DEV(port_id)
3377                 rte_eth_promiscuous_enable(port_id);
3378
3379         /* Init metrics library */
3380         rte_metrics_init(rte_socket_id());
3381
3382 #ifdef RTE_LIBRTE_LATENCY_STATS
3383         if (latencystats_enabled != 0) {
3384                 int ret = rte_latencystats_init(1, NULL);
3385                 if (ret)
3386                         printf("Warning: latencystats init()"
3387                                 " returned error %d\n", ret);
3388                 printf("Latencystats running on lcore %d\n",
3389                         latencystats_lcore_id);
3390         }
3391 #endif
3392
3393         /* Setup bitrate stats */
3394 #ifdef RTE_LIBRTE_BITRATE
3395         if (bitrate_enabled != 0) {
3396                 bitrate_data = rte_stats_bitrate_create();
3397                 if (bitrate_data == NULL)
3398                         rte_exit(EXIT_FAILURE,
3399                                 "Could not allocate bitrate data.\n");
3400                 rte_stats_bitrate_reg(bitrate_data);
3401         }
3402 #endif
3403
3404 #ifdef RTE_LIBRTE_CMDLINE
3405         if (strlen(cmdline_filename) != 0)
3406                 cmdline_read_from_file(cmdline_filename);
3407
3408         if (interactive == 1) {
3409                 if (auto_start) {
3410                         printf("Start automatic packet forwarding\n");
3411                         start_packet_forwarding(0);
3412                 }
3413                 prompt();
3414                 pmd_test_exit();
3415         } else
3416 #endif
3417         {
3418                 char c;
3419                 int rc;
3420
3421                 f_quit = 0;
3422
3423                 printf("No commandline core given, start packet forwarding\n");
3424                 start_packet_forwarding(tx_first);
3425                 if (stats_period != 0) {
3426                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3427                         uint64_t timer_period;
3428
3429                         /* Convert to number of cycles */
3430                         timer_period = stats_period * rte_get_timer_hz();
3431
3432                         while (f_quit == 0) {
3433                                 cur_time = rte_get_timer_cycles();
3434                                 diff_time += cur_time - prev_time;
3435
3436                                 if (diff_time >= timer_period) {
3437                                         print_stats();
3438                                         /* Reset the timer */
3439                                         diff_time = 0;
3440                                 }
3441                                 /* Sleep to avoid unnecessary checks */
3442                                 prev_time = cur_time;
3443                                 sleep(1);
3444                         }
3445                 }
3446
3447                 printf("Press enter to exit\n");
3448                 rc = read(0, &c, 1);
3449                 pmd_test_exit();
3450                 if (rc < 0)
3451                         return 1;
3452         }
3453
3454         return 0;
3455 }