app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_malloc_heap.h>
  31 #include <rte_memory.h>
  32 #include <rte_memcpy.h>
  33 #include <rte_launch.h>
  34 #include <rte_eal.h>
  35 #include <rte_alarm.h>
  36 #include <rte_per_lcore.h>
  37 #include <rte_lcore.h>
  38 #include <rte_atomic.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_LIBRTE_IXGBE_PMD
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIBRTE_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #include <rte_metrics.h>
  58 #ifdef RTE_LIBRTE_BITRATE
  59 #include <rte_bitrate.h>
  60 #endif
  61 #ifdef RTE_LIBRTE_LATENCY_STATS
  62 #include <rte_latencystats.h>
  63 #endif
  64
  65 #include "testpmd.h"
  66
  67 #ifndef MAP_HUGETLB
  68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  69 #define HUGE_FLAG (0x40000)
  70 #else
  71 #define HUGE_FLAG MAP_HUGETLB
  72 #endif
  73
  74 #ifndef MAP_HUGE_SHIFT
  75 /* older kernels (or FreeBSD) will not have this define */
  76 #define HUGE_SHIFT (26)
  77 #else
  78 #define HUGE_SHIFT MAP_HUGE_SHIFT
  79 #endif
  80
  81 #define EXTMEM_HEAP_NAME "extmem"
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192 uint16_t mempool_flags;
 193
 194 struct fwd_config cur_fwd_config;
 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 196 uint32_t retry_enabled;
 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 199
 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 201 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 202                                       * specified on command-line. */
 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 204
 205 /*
 206  * In container, it cannot terminate the process which running with 'stats-period'
 207  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 208  */
 209 uint8_t f_quit;
 210
 211 /*
 212  * Configuration of packet segments used by the "txonly" processing engine.
 213  */
 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 216         TXONLY_DEF_PACKET_LEN,
 217 };
 218 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 219
 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 221 /**< Split policy for packets to TX. */
 222
 223 uint8_t txonly_multi_flow;
 224 /**< Whether multiple flows are generated in TXONLY mode. */
 225
 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 228
 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 230 uint8_t dcb_config = 0;
 231
 232 /* Whether the dcb is in testing status */
 233 uint8_t dcb_test = 0;
 234
 235 /*
 236  * Configurable number of RX/TX queues.
 237  */
 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 240
 241 /*
 242  * Configurable number of RX/TX ring descriptors.
 243  * Defaults are supplied by drivers via ethdev.
 244  */
 245 #define RTE_TEST_RX_DESC_DEFAULT 0
 246 #define RTE_TEST_TX_DESC_DEFAULT 0
 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 249
 250 #define RTE_PMD_PARAM_UNSET -1
 251 /*
 252  * Configurable values of RX and TX ring threshold registers.
 253  */
 254
 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 258
 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 262
 263 /*
 264  * Configurable value of RX free threshold.
 265  */
 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 267
 268 /*
 269  * Configurable value of RX drop enable.
 270  */
 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 272
 273 /*
 274  * Configurable value of TX free threshold.
 275  */
 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of TX RS bit threshold.
 280  */
 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of buffered packets before sending.
 285  */
 286 uint16_t noisy_tx_sw_bufsz;
 287
 288 /*
 289  * Configurable value of packet buffer timeout.
 290  */
 291 uint16_t noisy_tx_sw_buf_flush_time;
 292
 293 /*
 294  * Configurable value for size of VNF internal memory area
 295  * used for simulating noisy neighbour behaviour
 296  */
 297 uint64_t noisy_lkup_mem_sz;
 298
 299 /*
 300  * Configurable value of number of random writes done in
 301  * VNF simulation memory area.
 302  */
 303 uint64_t noisy_lkup_num_writes;
 304
 305 /*
 306  * Configurable value of number of random reads done in
 307  * VNF simulation memory area.
 308  */
 309 uint64_t noisy_lkup_num_reads;
 310
 311 /*
 312  * Configurable value of number of random reads/writes done in
 313  * VNF simulation memory area.
 314  */
 315 uint64_t noisy_lkup_num_reads_writes;
 316
 317 /*
 318  * Receive Side Scaling (RSS) configuration.
 319  */
 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 321
 322 /*
 323  * Port topology configuration
 324  */
 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 326
 327 /*
 328  * Avoids to flush all the RX streams before starts forwarding.
 329  */
 330 uint8_t no_flush_rx = 0; /* flush by default */
 331
 332 /*
 333  * Flow API isolated mode.
 334  */
 335 uint8_t flow_isolate_all;
 336
 337 /*
 338  * Avoids to check link status when starting/stopping a port.
 339  */
 340 uint8_t no_link_check = 0; /* check by default */
 341
 342 /*
 343  * Enable link status change notification
 344  */
 345 uint8_t lsc_interrupt = 1; /* enabled by default */
 346
 347 /*
 348  * Enable device removal notification.
 349  */
 350 uint8_t rmv_interrupt = 1; /* enabled by default */
 351
 352 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 353
 354 /* After attach, port setup is called on event or by iterator */
 355 bool setup_on_probe_event = true;
 356
 357 /* Pretty printing of ethdev events */
 358 static const char * const eth_event_desc[] = {
 359         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 360         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 361         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 362         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 363         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 364         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 365         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 366         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 367         [RTE_ETH_EVENT_NEW] = "device probed",
 368         [RTE_ETH_EVENT_DESTROY] = "device released",
 369         [RTE_ETH_EVENT_MAX] = NULL,
 370 };
 371
 372 /*
 373  * Display or mask ether events
 374  * Default to all events except VF_MBOX
 375  */
 376 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 377                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 378                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 379                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 380                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 381                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 382                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 383 /*
 384  * Decide if all memory are locked for performance.
 385  */
 386 int do_mlockall = 0;
 387
 388 /*
 389  * NIC bypass mode configuration options.
 390  */
 391
 392 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 393 /* The NIC bypass watchdog timeout. */
 394 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 395 #endif
 396
 397
 398 #ifdef RTE_LIBRTE_LATENCY_STATS
 399
 400 /*
 401  * Set when latency stats is enabled in the commandline
 402  */
 403 uint8_t latencystats_enabled;
 404
 405 /*
 406  * Lcore ID to serive latency statistics.
 407  */
 408 lcoreid_t latencystats_lcore_id = -1;
 409
 410 #endif
 411
 412 /*
 413  * Ethernet device configuration.
 414  */
 415 struct rte_eth_rxmode rx_mode = {
 416         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 417 };
 418
 419 struct rte_eth_txmode tx_mode = {
 420         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 421 };
 422
 423 struct rte_fdir_conf fdir_conf = {
 424         .mode = RTE_FDIR_MODE_NONE,
 425         .pballoc = RTE_FDIR_PBALLOC_64K,
 426         .status = RTE_FDIR_REPORT_STATUS,
 427         .mask = {
 428                 .vlan_tci_mask = 0xFFEF,
 429                 .ipv4_mask     = {
 430                         .src_ip = 0xFFFFFFFF,
 431                         .dst_ip = 0xFFFFFFFF,
 432                 },
 433                 .ipv6_mask     = {
 434                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 435                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 436                 },
 437                 .src_port_mask = 0xFFFF,
 438                 .dst_port_mask = 0xFFFF,
 439                 .mac_addr_byte_mask = 0xFF,
 440                 .tunnel_type_mask = 1,
 441                 .tunnel_id_mask = 0xFFFFFFFF,
 442         },
 443         .drop_queue = 127,
 444 };
 445
 446 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 447
 448 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 449 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 450
 451 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 452 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 453
 454 uint16_t nb_tx_queue_stats_mappings = 0;
 455 uint16_t nb_rx_queue_stats_mappings = 0;
 456
 457 /*
 458  * Display zero values by default for xstats
 459  */
 460 uint8_t xstats_hide_zero;
 461
 462 unsigned int num_sockets = 0;
 463 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 464
 465 #ifdef RTE_LIBRTE_BITRATE
 466 /* Bitrate statistics */
 467 struct rte_stats_bitrates *bitrate_data;
 468 lcoreid_t bitrate_lcore_id;
 469 uint8_t bitrate_enabled;
 470 #endif
 471
 472 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 473 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 474
 475 struct vxlan_encap_conf vxlan_encap_conf = {
 476         .select_ipv4 = 1,
 477         .select_vlan = 0,
 478         .select_tos_ttl = 0,
 479         .vni = "\x00\x00\x00",
 480         .udp_src = 0,
 481         .udp_dst = RTE_BE16(4789),
 482         .ipv4_src = IPv4(127, 0, 0, 1),
 483         .ipv4_dst = IPv4(255, 255, 255, 255),
 484         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 485                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 486         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 487                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 488         .vlan_tci = 0,
 489         .ip_tos = 0,
 490         .ip_ttl = 255,
 491         .eth_src = "\x00\x00\x00\x00\x00\x00",
 492         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 493 };
 494
 495 struct nvgre_encap_conf nvgre_encap_conf = {
 496         .select_ipv4 = 1,
 497         .select_vlan = 0,
 498         .tni = "\x00\x00\x00",
 499         .ipv4_src = IPv4(127, 0, 0, 1),
 500         .ipv4_dst = IPv4(255, 255, 255, 255),
 501         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 502                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 503         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 504                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 505         .vlan_tci = 0,
 506         .eth_src = "\x00\x00\x00\x00\x00\x00",
 507         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 508 };
 509
 510 /* Forward function declarations */
 511 static void setup_attached_port(portid_t pi);
 512 static void map_port_queue_stats_mapping_registers(portid_t pi,
 513                                                    struct rte_port *port);
 514 static void check_all_ports_link_status(uint32_t port_mask);
 515 static int eth_event_callback(portid_t port_id,
 516                               enum rte_eth_event_type type,
 517                               void *param, void *ret_param);
 518 static void dev_event_callback(const char *device_name,
 519                                 enum rte_dev_event_type type,
 520                                 void *param);
 521
 522 /*
 523  * Check if all the ports are started.
 524  * If yes, return positive value. If not, return zero.
 525  */
 526 static int all_ports_started(void);
 527
 528 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 529 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN;
 530
 531 /*
 532  * Helper function to check if socket is already discovered.
 533  * If yes, return positive value. If not, return zero.
 534  */
 535 int
 536 new_socket_id(unsigned int socket_id)
 537 {
 538         unsigned int i;
 539
 540         for (i = 0; i < num_sockets; i++) {
 541                 if (socket_ids[i] == socket_id)
 542                         return 0;
 543         }
 544         return 1;
 545 }
 546
 547 /*
 548  * Setup default configuration.
 549  */
 550 static void
 551 set_default_fwd_lcores_config(void)
 552 {
 553         unsigned int i;
 554         unsigned int nb_lc;
 555         unsigned int sock_num;
 556
 557         nb_lc = 0;
 558         for (i = 0; i < RTE_MAX_LCORE; i++) {
 559                 if (!rte_lcore_is_enabled(i))
 560                         continue;
 561                 sock_num = rte_lcore_to_socket_id(i);
 562                 if (new_socket_id(sock_num)) {
 563                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 564                                 rte_exit(EXIT_FAILURE,
 565                                          "Total sockets greater than %u\n",
 566                                          RTE_MAX_NUMA_NODES);
 567                         }
 568                         socket_ids[num_sockets++] = sock_num;
 569                 }
 570                 if (i == rte_get_master_lcore())
 571                         continue;
 572                 fwd_lcores_cpuids[nb_lc++] = i;
 573         }
 574         nb_lcores = (lcoreid_t) nb_lc;
 575         nb_cfg_lcores = nb_lcores;
 576         nb_fwd_lcores = 1;
 577 }
 578
 579 static void
 580 set_def_peer_eth_addrs(void)
 581 {
 582         portid_t i;
 583
 584         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 585                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 586                 peer_eth_addrs[i].addr_bytes[5] = i;
 587         }
 588 }
 589
 590 static void
 591 set_default_fwd_ports_config(void)
 592 {
 593         portid_t pt_id;
 594         int i = 0;
 595
 596         RTE_ETH_FOREACH_DEV(pt_id) {
 597                 fwd_ports_ids[i++] = pt_id;
 598
 599                 /* Update sockets info according to the attached device */
 600                 int socket_id = rte_eth_dev_socket_id(pt_id);
 601                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 602                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 603                                 rte_exit(EXIT_FAILURE,
 604                                          "Total sockets greater than %u\n",
 605                                          RTE_MAX_NUMA_NODES);
 606                         }
 607                         socket_ids[num_sockets++] = socket_id;
 608                 }
 609         }
 610
 611         nb_cfg_ports = nb_ports;
 612         nb_fwd_ports = nb_ports;
 613 }
 614
 615 void
 616 set_def_fwd_config(void)
 617 {
 618         set_default_fwd_lcores_config();
 619         set_def_peer_eth_addrs();
 620         set_default_fwd_ports_config();
 621 }
 622
 623 /* extremely pessimistic estimation of memory required to create a mempool */
 624 static int
 625 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 626 {
 627         unsigned int n_pages, mbuf_per_pg, leftover;
 628         uint64_t total_mem, mbuf_mem, obj_sz;
 629
 630         /* there is no good way to predict how much space the mempool will
 631          * occupy because it will allocate chunks on the fly, and some of those
 632          * will come from default DPDK memory while some will come from our
 633          * external memory, so just assume 128MB will be enough for everyone.
 634          */
 635         uint64_t hdr_mem = 128 << 20;
 636
 637         /* account for possible non-contiguousness */
 638         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 639         if (obj_sz > pgsz) {
 640                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 641                 return -1;
 642         }
 643
 644         mbuf_per_pg = pgsz / obj_sz;
 645         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 646         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 647
 648         mbuf_mem = n_pages * pgsz;
 649
 650         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 651
 652         if (total_mem > SIZE_MAX) {
 653                 TESTPMD_LOG(ERR, "Memory size too big\n");
 654                 return -1;
 655         }
 656         *out = (size_t)total_mem;
 657
 658         return 0;
 659 }
 660
 661 static int
 662 pagesz_flags(uint64_t page_sz)
 663 {
 664         /* as per mmap() manpage, all page sizes are log2 of page size
 665          * shifted by MAP_HUGE_SHIFT
 666          */
 667         int log2 = rte_log2_u64(page_sz);
 668
 669         return (log2 << HUGE_SHIFT);
 670 }
 671
 672 static void *
 673 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 674 {
 675         void *addr;
 676         int flags;
 677
 678         /* allocate anonymous hugepages */
 679         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 680         if (huge)
 681                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 682
 683         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 684         if (addr == MAP_FAILED)
 685                 return NULL;
 686
 687         return addr;
 688 }
 689
 690 struct extmem_param {
 691         void *addr;
 692         size_t len;
 693         size_t pgsz;
 694         rte_iova_t *iova_table;
 695         unsigned int iova_table_len;
 696 };
 697
 698 static int
 699 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 700                 bool huge)
 701 {
 702         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 703                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 704         unsigned int cur_page, n_pages, pgsz_idx;
 705         size_t mem_sz, cur_pgsz;
 706         rte_iova_t *iovas = NULL;
 707         void *addr;
 708         int ret;
 709
 710         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 711                 /* skip anything that is too big */
 712                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 713                         continue;
 714
 715                 cur_pgsz = pgsizes[pgsz_idx];
 716
 717                 /* if we were told not to allocate hugepages, override */
 718                 if (!huge)
 719                         cur_pgsz = sysconf(_SC_PAGESIZE);
 720
 721                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 722                 if (ret < 0) {
 723                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 724                         return -1;
 725                 }
 726
 727                 /* allocate our memory */
 728                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 729
 730                 /* if we couldn't allocate memory with a specified page size,
 731                  * that doesn't mean we can't do it with other page sizes, so
 732                  * try another one.
 733                  */
 734                 if (addr == NULL)
 735                         continue;
 736
 737                 /* store IOVA addresses for every page in this memory area */
 738                 n_pages = mem_sz / cur_pgsz;
 739
 740                 iovas = malloc(sizeof(*iovas) * n_pages);
 741
 742                 if (iovas == NULL) {
 743                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 744                         goto fail;
 745                 }
 746                 /* lock memory if it's not huge pages */
 747                 if (!huge)
 748                         mlock(addr, mem_sz);
 749
 750                 /* populate IOVA addresses */
 751                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 752                         rte_iova_t iova;
 753                         size_t offset;
 754                         void *cur;
 755
 756                         offset = cur_pgsz * cur_page;
 757                         cur = RTE_PTR_ADD(addr, offset);
 758
 759                         /* touch the page before getting its IOVA */
 760                         *(volatile char *)cur = 0;
 761
 762                         iova = rte_mem_virt2iova(cur);
 763
 764                         iovas[cur_page] = iova;
 765                 }
 766
 767                 break;
 768         }
 769         /* if we couldn't allocate anything */
 770         if (iovas == NULL)
 771                 return -1;
 772
 773         param->addr = addr;
 774         param->len = mem_sz;
 775         param->pgsz = cur_pgsz;
 776         param->iova_table = iovas;
 777         param->iova_table_len = n_pages;
 778
 779         return 0;
 780 fail:
 781         if (iovas)
 782                 free(iovas);
 783         if (addr)
 784                 munmap(addr, mem_sz);
 785
 786         return -1;
 787 }
 788
 789 static int
 790 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 791 {
 792         struct extmem_param param;
 793         int socket_id, ret;
 794
 795         memset(&param, 0, sizeof(param));
 796
 797         /* check if our heap exists */
 798         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 799         if (socket_id < 0) {
 800                 /* create our heap */
 801                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 802                 if (ret < 0) {
 803                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 804                         return -1;
 805                 }
 806         }
 807
 808         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 809         if (ret < 0) {
 810                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 811                 return -1;
 812         }
 813
 814         /* we now have a valid memory area, so add it to heap */
 815         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 816                         param.addr, param.len, param.iova_table,
 817                         param.iova_table_len, param.pgsz);
 818
 819         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 820
 821         /* not needed any more */
 822         free(param.iova_table);
 823
 824         if (ret < 0) {
 825                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 826                 munmap(param.addr, param.len);
 827                 return -1;
 828         }
 829
 830         /* success */
 831
 832         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 833                         param.len >> 20);
 834
 835         return 0;
 836 }
 837
 838 /*
 839  * Configuration initialisation done once at init time.
 840  */
 841 static struct rte_mempool *
 842 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 843                  unsigned int socket_id)
 844 {
 845         char pool_name[RTE_MEMPOOL_NAMESIZE];
 846         struct rte_mempool *rte_mp = NULL;
 847         uint32_t mb_size;
 848
 849         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 850         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 851
 852         TESTPMD_LOG(INFO,
 853                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 854                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 855
 856         switch (mp_alloc_type) {
 857         case MP_ALLOC_NATIVE:
 858                 {
 859                         /* wrapper to rte_mempool_create() */
 860                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 861                                         rte_mbuf_best_mempool_ops());
 862                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 863                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 864                         break;
 865                 }
 866         case MP_ALLOC_ANON:
 867                 {
 868                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 869                                 mb_size, (unsigned int) mb_mempool_cache,
 870                                 sizeof(struct rte_pktmbuf_pool_private),
 871                                 socket_id, mempool_flags);
 872                         if (rte_mp == NULL)
 873                                 goto err;
 874
 875                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 876                                 rte_mempool_free(rte_mp);
 877                                 rte_mp = NULL;
 878                                 goto err;
 879                         }
 880                         rte_pktmbuf_pool_init(rte_mp, NULL);
 881                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 882                         break;
 883                 }
 884         case MP_ALLOC_XMEM:
 885         case MP_ALLOC_XMEM_HUGE:
 886                 {
 887                         int heap_socket;
 888                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 889
 890                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 891                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 892
 893                         heap_socket =
 894                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 895                         if (heap_socket < 0)
 896                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 897
 898                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 899                                         rte_mbuf_best_mempool_ops());
 900                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 901                                         mb_mempool_cache, 0, mbuf_seg_size,
 902                                         heap_socket);
 903                         break;
 904                 }
 905         default:
 906                 {
 907                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 908                 }
 909         }
 910
 911 err:
 912         if (rte_mp == NULL) {
 913                 rte_exit(EXIT_FAILURE,
 914                         "Creation of mbuf pool for socket %u failed: %s\n",
 915                         socket_id, rte_strerror(rte_errno));
 916         } else if (verbose_level > 0) {
 917                 rte_mempool_dump(stdout, rte_mp);
 918         }
 919         return rte_mp;
 920 }
 921
 922 /*
 923  * Check given socket id is valid or not with NUMA mode,
 924  * if valid, return 0, else return -1
 925  */
 926 static int
 927 check_socket_id(const unsigned int socket_id)
 928 {
 929         static int warning_once = 0;
 930
 931         if (new_socket_id(socket_id)) {
 932                 if (!warning_once && numa_support)
 933                         printf("Warning: NUMA should be configured manually by"
 934                                " using --port-numa-config and"
 935                                " --ring-numa-config parameters along with"
 936                                " --numa.\n");
 937                 warning_once = 1;
 938                 return -1;
 939         }
 940         return 0;
 941 }
 942
 943 /*
 944  * Get the allowed maximum number of RX queues.
 945  * *pid return the port id which has minimal value of
 946  * max_rx_queues in all ports.
 947  */
 948 queueid_t
 949 get_allowed_max_nb_rxq(portid_t *pid)
 950 {
 951         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
 952         portid_t pi;
 953         struct rte_eth_dev_info dev_info;
 954
 955         RTE_ETH_FOREACH_DEV(pi) {
 956                 rte_eth_dev_info_get(pi, &dev_info);
 957                 if (dev_info.max_rx_queues < allowed_max_rxq) {
 958                         allowed_max_rxq = dev_info.max_rx_queues;
 959                         *pid = pi;
 960                 }
 961         }
 962         return allowed_max_rxq;
 963 }
 964
 965 /*
 966  * Check input rxq is valid or not.
 967  * If input rxq is not greater than any of maximum number
 968  * of RX queues of all ports, it is valid.
 969  * if valid, return 0, else return -1
 970  */
 971 int
 972 check_nb_rxq(queueid_t rxq)
 973 {
 974         queueid_t allowed_max_rxq;
 975         portid_t pid = 0;
 976
 977         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
 978         if (rxq > allowed_max_rxq) {
 979                 printf("Fail: input rxq (%u) can't be greater "
 980                        "than max_rx_queues (%u) of port %u\n",
 981                        rxq,
 982                        allowed_max_rxq,
 983                        pid);
 984                 return -1;
 985         }
 986         return 0;
 987 }
 988
 989 /*
 990  * Get the allowed maximum number of TX queues.
 991  * *pid return the port id which has minimal value of
 992  * max_tx_queues in all ports.
 993  */
 994 queueid_t
 995 get_allowed_max_nb_txq(portid_t *pid)
 996 {
 997         queueid_t allowed_max_txq = MAX_QUEUE_ID;
 998         portid_t pi;
 999         struct rte_eth_dev_info dev_info;
1000
1001         RTE_ETH_FOREACH_DEV(pi) {
1002                 rte_eth_dev_info_get(pi, &dev_info);
1003                 if (dev_info.max_tx_queues < allowed_max_txq) {
1004                         allowed_max_txq = dev_info.max_tx_queues;
1005                         *pid = pi;
1006                 }
1007         }
1008         return allowed_max_txq;
1009 }
1010
1011 /*
1012  * Check input txq is valid or not.
1013  * If input txq is not greater than any of maximum number
1014  * of TX queues of all ports, it is valid.
1015  * if valid, return 0, else return -1
1016  */
1017 int
1018 check_nb_txq(queueid_t txq)
1019 {
1020         queueid_t allowed_max_txq;
1021         portid_t pid = 0;
1022
1023         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1024         if (txq > allowed_max_txq) {
1025                 printf("Fail: input txq (%u) can't be greater "
1026                        "than max_tx_queues (%u) of port %u\n",
1027                        txq,
1028                        allowed_max_txq,
1029                        pid);
1030                 return -1;
1031         }
1032         return 0;
1033 }
1034
1035 static void
1036 init_config(void)
1037 {
1038         portid_t pid;
1039         struct rte_port *port;
1040         struct rte_mempool *mbp;
1041         unsigned int nb_mbuf_per_pool;
1042         lcoreid_t  lc_id;
1043         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1044         struct rte_gro_param gro_param;
1045         uint32_t gso_types;
1046         int k;
1047
1048         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1049
1050         /* Configuration of logical cores. */
1051         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1052                                 sizeof(struct fwd_lcore *) * nb_lcores,
1053                                 RTE_CACHE_LINE_SIZE);
1054         if (fwd_lcores == NULL) {
1055                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1056                                                         "failed\n", nb_lcores);
1057         }
1058         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1059                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1060                                                sizeof(struct fwd_lcore),
1061                                                RTE_CACHE_LINE_SIZE);
1062                 if (fwd_lcores[lc_id] == NULL) {
1063                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1064                                                                 "failed\n");
1065                 }
1066                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1067         }
1068
1069         RTE_ETH_FOREACH_DEV(pid) {
1070                 port = &ports[pid];
1071                 /* Apply default TxRx configuration for all ports */
1072                 port->dev_conf.txmode = tx_mode;
1073                 port->dev_conf.rxmode = rx_mode;
1074                 rte_eth_dev_info_get(pid, &port->dev_info);
1075
1076                 if (!(port->dev_info.tx_offload_capa &
1077                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1078                         port->dev_conf.txmode.offloads &=
1079                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1080                 if (!(port->dev_info.tx_offload_capa &
1081                         DEV_TX_OFFLOAD_MATCH_METADATA))
1082                         port->dev_conf.txmode.offloads &=
1083                                 ~DEV_TX_OFFLOAD_MATCH_METADATA;
1084                 if (numa_support) {
1085                         if (port_numa[pid] != NUMA_NO_CONFIG)
1086                                 port_per_socket[port_numa[pid]]++;
1087                         else {
1088                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1089
1090                                 /*
1091                                  * if socket_id is invalid,
1092                                  * set to the first available socket.
1093                                  */
1094                                 if (check_socket_id(socket_id) < 0)
1095                                         socket_id = socket_ids[0];
1096                                 port_per_socket[socket_id]++;
1097                         }
1098                 }
1099
1100                 /* Apply Rx offloads configuration */
1101                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1102                         port->rx_conf[k].offloads =
1103                                 port->dev_conf.rxmode.offloads;
1104                 /* Apply Tx offloads configuration */
1105                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1106                         port->tx_conf[k].offloads =
1107                                 port->dev_conf.txmode.offloads;
1108
1109                 /* set flag to initialize port/queue */
1110                 port->need_reconfig = 1;
1111                 port->need_reconfig_queues = 1;
1112                 port->tx_metadata = 0;
1113         }
1114
1115         /*
1116          * Create pools of mbuf.
1117          * If NUMA support is disabled, create a single pool of mbuf in
1118          * socket 0 memory by default.
1119          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1120          *
1121          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1122          * nb_txd can be configured at run time.
1123          */
1124         if (param_total_num_mbufs)
1125                 nb_mbuf_per_pool = param_total_num_mbufs;
1126         else {
1127                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1128                         (nb_lcores * mb_mempool_cache) +
1129                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1130                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1131         }
1132
1133         if (numa_support) {
1134                 uint8_t i;
1135
1136                 for (i = 0; i < num_sockets; i++)
1137                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1138                                                        nb_mbuf_per_pool,
1139                                                        socket_ids[i]);
1140         } else {
1141                 if (socket_num == UMA_NO_CONFIG)
1142                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1143                                                        nb_mbuf_per_pool, 0);
1144                 else
1145                         mempools[socket_num] = mbuf_pool_create
1146                                                         (mbuf_data_size,
1147                                                          nb_mbuf_per_pool,
1148                                                          socket_num);
1149         }
1150
1151         init_port_config();
1152
1153         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1154                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1155         /*
1156          * Records which Mbuf pool to use by each logical core, if needed.
1157          */
1158         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1159                 mbp = mbuf_pool_find(
1160                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1161
1162                 if (mbp == NULL)
1163                         mbp = mbuf_pool_find(0);
1164                 fwd_lcores[lc_id]->mbp = mbp;
1165                 /* initialize GSO context */
1166                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1167                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1168                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1169                 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN -
1170                         ETHER_CRC_LEN;
1171                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1172         }
1173
1174         /* Configuration of packet forwarding streams. */
1175         if (init_fwd_streams() < 0)
1176                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1177
1178         fwd_config_setup();
1179
1180         /* create a gro context for each lcore */
1181         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1182         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1183         gro_param.max_item_per_flow = MAX_PKT_BURST;
1184         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1185                 gro_param.socket_id = rte_lcore_to_socket_id(
1186                                 fwd_lcores_cpuids[lc_id]);
1187                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1188                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1189                         rte_exit(EXIT_FAILURE,
1190                                         "rte_gro_ctx_create() failed\n");
1191                 }
1192         }
1193
1194 #if defined RTE_LIBRTE_PMD_SOFTNIC
1195         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1196                 RTE_ETH_FOREACH_DEV(pid) {
1197                         port = &ports[pid];
1198                         const char *driver = port->dev_info.driver_name;
1199
1200                         if (strcmp(driver, "net_softnic") == 0)
1201                                 port->softport.fwd_lcore_arg = fwd_lcores;
1202                 }
1203         }
1204 #endif
1205
1206 }
1207
1208
1209 void
1210 reconfig(portid_t new_port_id, unsigned socket_id)
1211 {
1212         struct rte_port *port;
1213
1214         /* Reconfiguration of Ethernet ports. */
1215         port = &ports[new_port_id];
1216         rte_eth_dev_info_get(new_port_id, &port->dev_info);
1217
1218         /* set flag to initialize port/queue */
1219         port->need_reconfig = 1;
1220         port->need_reconfig_queues = 1;
1221         port->socket_id = socket_id;
1222
1223         init_port_config();
1224 }
1225
1226
1227 int
1228 init_fwd_streams(void)
1229 {
1230         portid_t pid;
1231         struct rte_port *port;
1232         streamid_t sm_id, nb_fwd_streams_new;
1233         queueid_t q;
1234
1235         /* set socket id according to numa or not */
1236         RTE_ETH_FOREACH_DEV(pid) {
1237                 port = &ports[pid];
1238                 if (nb_rxq > port->dev_info.max_rx_queues) {
1239                         printf("Fail: nb_rxq(%d) is greater than "
1240                                 "max_rx_queues(%d)\n", nb_rxq,
1241                                 port->dev_info.max_rx_queues);
1242                         return -1;
1243                 }
1244                 if (nb_txq > port->dev_info.max_tx_queues) {
1245                         printf("Fail: nb_txq(%d) is greater than "
1246                                 "max_tx_queues(%d)\n", nb_txq,
1247                                 port->dev_info.max_tx_queues);
1248                         return -1;
1249                 }
1250                 if (numa_support) {
1251                         if (port_numa[pid] != NUMA_NO_CONFIG)
1252                                 port->socket_id = port_numa[pid];
1253                         else {
1254                                 port->socket_id = rte_eth_dev_socket_id(pid);
1255
1256                                 /*
1257                                  * if socket_id is invalid,
1258                                  * set to the first available socket.
1259                                  */
1260                                 if (check_socket_id(port->socket_id) < 0)
1261                                         port->socket_id = socket_ids[0];
1262                         }
1263                 }
1264                 else {
1265                         if (socket_num == UMA_NO_CONFIG)
1266                                 port->socket_id = 0;
1267                         else
1268                                 port->socket_id = socket_num;
1269                 }
1270         }
1271
1272         q = RTE_MAX(nb_rxq, nb_txq);
1273         if (q == 0) {
1274                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1275                 return -1;
1276         }
1277         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1278         if (nb_fwd_streams_new == nb_fwd_streams)
1279                 return 0;
1280         /* clear the old */
1281         if (fwd_streams != NULL) {
1282                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1283                         if (fwd_streams[sm_id] == NULL)
1284                                 continue;
1285                         rte_free(fwd_streams[sm_id]);
1286                         fwd_streams[sm_id] = NULL;
1287                 }
1288                 rte_free(fwd_streams);
1289                 fwd_streams = NULL;
1290         }
1291
1292         /* init new */
1293         nb_fwd_streams = nb_fwd_streams_new;
1294         if (nb_fwd_streams) {
1295                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1296                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1297                         RTE_CACHE_LINE_SIZE);
1298                 if (fwd_streams == NULL)
1299                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1300                                  " (struct fwd_stream *)) failed\n",
1301                                  nb_fwd_streams);
1302
1303                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1304                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1305                                 " struct fwd_stream", sizeof(struct fwd_stream),
1306                                 RTE_CACHE_LINE_SIZE);
1307                         if (fwd_streams[sm_id] == NULL)
1308                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1309                                          "(struct fwd_stream) failed\n");
1310                 }
1311         }
1312
1313         return 0;
1314 }
1315
1316 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1317 static void
1318 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1319 {
1320         unsigned int total_burst;
1321         unsigned int nb_burst;
1322         unsigned int burst_stats[3];
1323         uint16_t pktnb_stats[3];
1324         uint16_t nb_pkt;
1325         int burst_percent[3];
1326
1327         /*
1328          * First compute the total number of packet bursts and the
1329          * two highest numbers of bursts of the same number of packets.
1330          */
1331         total_burst = 0;
1332         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1333         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1334         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1335                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1336                 if (nb_burst == 0)
1337                         continue;
1338                 total_burst += nb_burst;
1339                 if (nb_burst > burst_stats[0]) {
1340                         burst_stats[1] = burst_stats[0];
1341                         pktnb_stats[1] = pktnb_stats[0];
1342                         burst_stats[0] = nb_burst;
1343                         pktnb_stats[0] = nb_pkt;
1344                 } else if (nb_burst > burst_stats[1]) {
1345                         burst_stats[1] = nb_burst;
1346                         pktnb_stats[1] = nb_pkt;
1347                 }
1348         }
1349         if (total_burst == 0)
1350                 return;
1351         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1352         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1353                burst_percent[0], (int) pktnb_stats[0]);
1354         if (burst_stats[0] == total_burst) {
1355                 printf("]\n");
1356                 return;
1357         }
1358         if (burst_stats[0] + burst_stats[1] == total_burst) {
1359                 printf(" + %d%% of %d pkts]\n",
1360                        100 - burst_percent[0], pktnb_stats[1]);
1361                 return;
1362         }
1363         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1364         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1365         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1366                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1367                 return;
1368         }
1369         printf(" + %d%% of %d pkts + %d%% of others]\n",
1370                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1371 }
1372 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1373
1374 static void
1375 fwd_stream_stats_display(streamid_t stream_id)
1376 {
1377         struct fwd_stream *fs;
1378         static const char *fwd_top_stats_border = "-------";
1379
1380         fs = fwd_streams[stream_id];
1381         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1382             (fs->fwd_dropped == 0))
1383                 return;
1384         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1385                "TX Port=%2d/Queue=%2d %s\n",
1386                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1387                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1388         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1389                " TX-dropped: %-14"PRIu64,
1390                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1391
1392         /* if checksum mode */
1393         if (cur_fwd_eng == &csum_fwd_engine) {
1394                 printf("  RX- bad IP checksum: %-14"PRIu64
1395                        "  Rx- bad L4 checksum: %-14"PRIu64
1396                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1397                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1398                         fs->rx_bad_outer_l4_csum);
1399         } else {
1400                 printf("\n");
1401         }
1402
1403 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1404         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1405         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1406 #endif
1407 }
1408
1409 void
1410 fwd_stats_display(void)
1411 {
1412         static const char *fwd_stats_border = "----------------------";
1413         static const char *acc_stats_border = "+++++++++++++++";
1414         struct {
1415                 struct fwd_stream *rx_stream;
1416                 struct fwd_stream *tx_stream;
1417                 uint64_t tx_dropped;
1418                 uint64_t rx_bad_ip_csum;
1419                 uint64_t rx_bad_l4_csum;
1420                 uint64_t rx_bad_outer_l4_csum;
1421         } ports_stats[RTE_MAX_ETHPORTS];
1422         uint64_t total_rx_dropped = 0;
1423         uint64_t total_tx_dropped = 0;
1424         uint64_t total_rx_nombuf = 0;
1425         struct rte_eth_stats stats;
1426 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1427         uint64_t fwd_cycles = 0;
1428 #endif
1429         uint64_t total_recv = 0;
1430         uint64_t total_xmit = 0;
1431         struct rte_port *port;
1432         streamid_t sm_id;
1433         portid_t pt_id;
1434         int i;
1435
1436         memset(ports_stats, 0, sizeof(ports_stats));
1437
1438         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1439                 struct fwd_stream *fs = fwd_streams[sm_id];
1440
1441                 if (cur_fwd_config.nb_fwd_streams >
1442                     cur_fwd_config.nb_fwd_ports) {
1443                         fwd_stream_stats_display(sm_id);
1444                 } else {
1445                         ports_stats[fs->tx_port].tx_stream = fs;
1446                         ports_stats[fs->rx_port].rx_stream = fs;
1447                 }
1448
1449                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1450
1451                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1452                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1453                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1454                                 fs->rx_bad_outer_l4_csum;
1455
1456 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1457                 fwd_cycles += fs->core_cycles;
1458 #endif
1459         }
1460         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1461                 uint8_t j;
1462
1463                 pt_id = fwd_ports_ids[i];
1464                 port = &ports[pt_id];
1465
1466                 rte_eth_stats_get(pt_id, &stats);
1467                 stats.ipackets -= port->stats.ipackets;
1468                 stats.opackets -= port->stats.opackets;
1469                 stats.ibytes -= port->stats.ibytes;
1470                 stats.obytes -= port->stats.obytes;
1471                 stats.imissed -= port->stats.imissed;
1472                 stats.oerrors -= port->stats.oerrors;
1473                 stats.rx_nombuf -= port->stats.rx_nombuf;
1474
1475                 total_recv += stats.ipackets;
1476                 total_xmit += stats.opackets;
1477                 total_rx_dropped += stats.imissed;
1478                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1479                 total_tx_dropped += stats.oerrors;
1480                 total_rx_nombuf  += stats.rx_nombuf;
1481
1482                 printf("\n  %s Forward statistics for port %-2d %s\n",
1483                        fwd_stats_border, pt_id, fwd_stats_border);
1484
1485                 if (!port->rx_queue_stats_mapping_enabled &&
1486                     !port->tx_queue_stats_mapping_enabled) {
1487                         printf("  RX-packets: %-14"PRIu64
1488                                " RX-dropped: %-14"PRIu64
1489                                "RX-total: %-"PRIu64"\n",
1490                                stats.ipackets, stats.imissed,
1491                                stats.ipackets + stats.imissed);
1492
1493                         if (cur_fwd_eng == &csum_fwd_engine)
1494                                 printf("  Bad-ipcsum: %-14"PRIu64
1495                                        " Bad-l4csum: %-14"PRIu64
1496                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1497                                        ports_stats[pt_id].rx_bad_ip_csum,
1498                                        ports_stats[pt_id].rx_bad_l4_csum,
1499                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1500                         if (stats.ierrors + stats.rx_nombuf > 0) {
1501                                 printf("  RX-error: %-"PRIu64"\n",
1502                                        stats.ierrors);
1503                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1504                                        stats.rx_nombuf);
1505                         }
1506
1507                         printf("  TX-packets: %-14"PRIu64
1508                                " TX-dropped: %-14"PRIu64
1509                                "TX-total: %-"PRIu64"\n",
1510                                stats.opackets, ports_stats[pt_id].tx_dropped,
1511                                stats.opackets + ports_stats[pt_id].tx_dropped);
1512                 } else {
1513                         printf("  RX-packets:             %14"PRIu64
1514                                "    RX-dropped:%14"PRIu64
1515                                "    RX-total:%14"PRIu64"\n",
1516                                stats.ipackets, stats.imissed,
1517                                stats.ipackets + stats.imissed);
1518
1519                         if (cur_fwd_eng == &csum_fwd_engine)
1520                                 printf("  Bad-ipcsum:%14"PRIu64
1521                                        "    Bad-l4csum:%14"PRIu64
1522                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1523                                        ports_stats[pt_id].rx_bad_ip_csum,
1524                                        ports_stats[pt_id].rx_bad_l4_csum,
1525                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1526                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1527                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1528                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1529                                        stats.rx_nombuf);
1530                         }
1531
1532                         printf("  TX-packets:             %14"PRIu64
1533                                "    TX-dropped:%14"PRIu64
1534                                "    TX-total:%14"PRIu64"\n",
1535                                stats.opackets, ports_stats[pt_id].tx_dropped,
1536                                stats.opackets + ports_stats[pt_id].tx_dropped);
1537                 }
1538
1539 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1540                 if (ports_stats[pt_id].rx_stream)
1541                         pkt_burst_stats_display("RX",
1542                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1543                 if (ports_stats[pt_id].tx_stream)
1544                         pkt_burst_stats_display("TX",
1545                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1546 #endif
1547
1548                 if (port->rx_queue_stats_mapping_enabled) {
1549                         printf("\n");
1550                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1551                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1552                                        "     RX-errors:%14"PRIu64
1553                                        "    RX-bytes:%14"PRIu64"\n",
1554                                        j, stats.q_ipackets[j],
1555                                        stats.q_errors[j], stats.q_ibytes[j]);
1556                         }
1557                         printf("\n");
1558                 }
1559                 if (port->tx_queue_stats_mapping_enabled) {
1560                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1561                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1562                                        "                                 TX-bytes:%14"
1563                                        PRIu64"\n",
1564                                        j, stats.q_opackets[j],
1565                                        stats.q_obytes[j]);
1566                         }
1567                 }
1568
1569                 printf("  %s--------------------------------%s\n",
1570                        fwd_stats_border, fwd_stats_border);
1571         }
1572
1573         printf("\n  %s Accumulated forward statistics for all ports"
1574                "%s\n",
1575                acc_stats_border, acc_stats_border);
1576         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1577                "%-"PRIu64"\n"
1578                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1579                "%-"PRIu64"\n",
1580                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1581                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1582         if (total_rx_nombuf > 0)
1583                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1584         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1585                "%s\n",
1586                acc_stats_border, acc_stats_border);
1587 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1588         if (total_recv > 0)
1589                 printf("\n  CPU cycles/packet=%u (total cycles="
1590                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1591                        (unsigned int)(fwd_cycles / total_recv),
1592                        fwd_cycles, total_recv);
1593 #endif
1594 }
1595
1596 void
1597 fwd_stats_reset(void)
1598 {
1599         streamid_t sm_id;
1600         portid_t pt_id;
1601         int i;
1602
1603         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1604                 pt_id = fwd_ports_ids[i];
1605                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1606         }
1607         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1608                 struct fwd_stream *fs = fwd_streams[sm_id];
1609
1610                 fs->rx_packets = 0;
1611                 fs->tx_packets = 0;
1612                 fs->fwd_dropped = 0;
1613                 fs->rx_bad_ip_csum = 0;
1614                 fs->rx_bad_l4_csum = 0;
1615                 fs->rx_bad_outer_l4_csum = 0;
1616
1617 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1618                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1619                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1620 #endif
1621 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1622                 fs->core_cycles = 0;
1623 #endif
1624         }
1625 }
1626
1627 static void
1628 flush_fwd_rx_queues(void)
1629 {
1630         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1631         portid_t  rxp;
1632         portid_t port_id;
1633         queueid_t rxq;
1634         uint16_t  nb_rx;
1635         uint16_t  i;
1636         uint8_t   j;
1637         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1638         uint64_t timer_period;
1639
1640         /* convert to number of cycles */
1641         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1642
1643         for (j = 0; j < 2; j++) {
1644                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1645                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1646                                 port_id = fwd_ports_ids[rxp];
1647                                 /**
1648                                 * testpmd can stuck in the below do while loop
1649                                 * if rte_eth_rx_burst() always returns nonzero
1650                                 * packets. So timer is added to exit this loop
1651                                 * after 1sec timer expiry.
1652                                 */
1653                                 prev_tsc = rte_rdtsc();
1654                                 do {
1655                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1656                                                 pkts_burst, MAX_PKT_BURST);
1657                                         for (i = 0; i < nb_rx; i++)
1658                                                 rte_pktmbuf_free(pkts_burst[i]);
1659
1660                                         cur_tsc = rte_rdtsc();
1661                                         diff_tsc = cur_tsc - prev_tsc;
1662                                         timer_tsc += diff_tsc;
1663                                 } while ((nb_rx > 0) &&
1664                                         (timer_tsc < timer_period));
1665                                 timer_tsc = 0;
1666                         }
1667                 }
1668                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1669         }
1670 }
1671
1672 static void
1673 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1674 {
1675         struct fwd_stream **fsm;
1676         streamid_t nb_fs;
1677         streamid_t sm_id;
1678 #ifdef RTE_LIBRTE_BITRATE
1679         uint64_t tics_per_1sec;
1680         uint64_t tics_datum;
1681         uint64_t tics_current;
1682         uint16_t i, cnt_ports;
1683
1684         cnt_ports = nb_ports;
1685         tics_datum = rte_rdtsc();
1686         tics_per_1sec = rte_get_timer_hz();
1687 #endif
1688         fsm = &fwd_streams[fc->stream_idx];
1689         nb_fs = fc->stream_nb;
1690         do {
1691                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1692                         (*pkt_fwd)(fsm[sm_id]);
1693 #ifdef RTE_LIBRTE_BITRATE
1694                 if (bitrate_enabled != 0 &&
1695                                 bitrate_lcore_id == rte_lcore_id()) {
1696                         tics_current = rte_rdtsc();
1697                         if (tics_current - tics_datum >= tics_per_1sec) {
1698                                 /* Periodic bitrate calculation */
1699                                 for (i = 0; i < cnt_ports; i++)
1700                                         rte_stats_bitrate_calc(bitrate_data,
1701                                                 ports_ids[i]);
1702                                 tics_datum = tics_current;
1703                         }
1704                 }
1705 #endif
1706 #ifdef RTE_LIBRTE_LATENCY_STATS
1707                 if (latencystats_enabled != 0 &&
1708                                 latencystats_lcore_id == rte_lcore_id())
1709                         rte_latencystats_update();
1710 #endif
1711
1712         } while (! fc->stopped);
1713 }
1714
1715 static int
1716 start_pkt_forward_on_core(void *fwd_arg)
1717 {
1718         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1719                              cur_fwd_config.fwd_eng->packet_fwd);
1720         return 0;
1721 }
1722
1723 /*
1724  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1725  * Used to start communication flows in network loopback test configurations.
1726  */
1727 static int
1728 run_one_txonly_burst_on_core(void *fwd_arg)
1729 {
1730         struct fwd_lcore *fwd_lc;
1731         struct fwd_lcore tmp_lcore;
1732
1733         fwd_lc = (struct fwd_lcore *) fwd_arg;
1734         tmp_lcore = *fwd_lc;
1735         tmp_lcore.stopped = 1;
1736         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1737         return 0;
1738 }
1739
1740 /*
1741  * Launch packet forwarding:
1742  *     - Setup per-port forwarding context.
1743  *     - launch logical cores with their forwarding configuration.
1744  */
1745 static void
1746 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1747 {
1748         port_fwd_begin_t port_fwd_begin;
1749         unsigned int i;
1750         unsigned int lc_id;
1751         int diag;
1752
1753         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1754         if (port_fwd_begin != NULL) {
1755                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1756                         (*port_fwd_begin)(fwd_ports_ids[i]);
1757         }
1758         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1759                 lc_id = fwd_lcores_cpuids[i];
1760                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1761                         fwd_lcores[i]->stopped = 0;
1762                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1763                                                      fwd_lcores[i], lc_id);
1764                         if (diag != 0)
1765                                 printf("launch lcore %u failed - diag=%d\n",
1766                                        lc_id, diag);
1767                 }
1768         }
1769 }
1770
1771 /*
1772  * Launch packet forwarding configuration.
1773  */
1774 void
1775 start_packet_forwarding(int with_tx_first)
1776 {
1777         port_fwd_begin_t port_fwd_begin;
1778         port_fwd_end_t  port_fwd_end;
1779         struct rte_port *port;
1780         unsigned int i;
1781         portid_t   pt_id;
1782
1783         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1784                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1785
1786         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1787                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1788
1789         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1790                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1791                 (!nb_rxq || !nb_txq))
1792                 rte_exit(EXIT_FAILURE,
1793                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1794                         cur_fwd_eng->fwd_mode_name);
1795
1796         if (all_ports_started() == 0) {
1797                 printf("Not all ports were started\n");
1798                 return;
1799         }
1800         if (test_done == 0) {
1801                 printf("Packet forwarding already started\n");
1802                 return;
1803         }
1804
1805
1806         if(dcb_test) {
1807                 for (i = 0; i < nb_fwd_ports; i++) {
1808                         pt_id = fwd_ports_ids[i];
1809                         port = &ports[pt_id];
1810                         if (!port->dcb_flag) {
1811                                 printf("In DCB mode, all forwarding ports must "
1812                                        "be configured in this mode.\n");
1813                                 return;
1814                         }
1815                 }
1816                 if (nb_fwd_lcores == 1) {
1817                         printf("In DCB mode,the nb forwarding cores "
1818                                "should be larger than 1.\n");
1819                         return;
1820                 }
1821         }
1822         test_done = 0;
1823
1824         fwd_config_setup();
1825
1826         if(!no_flush_rx)
1827                 flush_fwd_rx_queues();
1828
1829         pkt_fwd_config_display(&cur_fwd_config);
1830         rxtx_config_display();
1831
1832         fwd_stats_reset();
1833         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1834                 pt_id = fwd_ports_ids[i];
1835                 port = &ports[pt_id];
1836                 map_port_queue_stats_mapping_registers(pt_id, port);
1837         }
1838         if (with_tx_first) {
1839                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1840                 if (port_fwd_begin != NULL) {
1841                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1842                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1843                 }
1844                 while (with_tx_first--) {
1845                         launch_packet_forwarding(
1846                                         run_one_txonly_burst_on_core);
1847                         rte_eal_mp_wait_lcore();
1848                 }
1849                 port_fwd_end = tx_only_engine.port_fwd_end;
1850                 if (port_fwd_end != NULL) {
1851                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1852                                 (*port_fwd_end)(fwd_ports_ids[i]);
1853                 }
1854         }
1855         launch_packet_forwarding(start_pkt_forward_on_core);
1856 }
1857
1858 void
1859 stop_packet_forwarding(void)
1860 {
1861         port_fwd_end_t port_fwd_end;
1862         lcoreid_t lc_id;
1863         portid_t pt_id;
1864         int i;
1865
1866         if (test_done) {
1867                 printf("Packet forwarding not started\n");
1868                 return;
1869         }
1870         printf("Telling cores to stop...");
1871         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1872                 fwd_lcores[lc_id]->stopped = 1;
1873         printf("\nWaiting for lcores to finish...\n");
1874         rte_eal_mp_wait_lcore();
1875         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1876         if (port_fwd_end != NULL) {
1877                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1878                         pt_id = fwd_ports_ids[i];
1879                         (*port_fwd_end)(pt_id);
1880                 }
1881         }
1882
1883         fwd_stats_display();
1884
1885         printf("\nDone.\n");
1886         test_done = 1;
1887 }
1888
1889 void
1890 dev_set_link_up(portid_t pid)
1891 {
1892         if (rte_eth_dev_set_link_up(pid) < 0)
1893                 printf("\nSet link up fail.\n");
1894 }
1895
1896 void
1897 dev_set_link_down(portid_t pid)
1898 {
1899         if (rte_eth_dev_set_link_down(pid) < 0)
1900                 printf("\nSet link down fail.\n");
1901 }
1902
1903 static int
1904 all_ports_started(void)
1905 {
1906         portid_t pi;
1907         struct rte_port *port;
1908
1909         RTE_ETH_FOREACH_DEV(pi) {
1910                 port = &ports[pi];
1911                 /* Check if there is a port which is not started */
1912                 if ((port->port_status != RTE_PORT_STARTED) &&
1913                         (port->slave_flag == 0))
1914                         return 0;
1915         }
1916
1917         /* No port is not started */
1918         return 1;
1919 }
1920
1921 int
1922 port_is_stopped(portid_t port_id)
1923 {
1924         struct rte_port *port = &ports[port_id];
1925
1926         if ((port->port_status != RTE_PORT_STOPPED) &&
1927             (port->slave_flag == 0))
1928                 return 0;
1929         return 1;
1930 }
1931
1932 int
1933 all_ports_stopped(void)
1934 {
1935         portid_t pi;
1936
1937         RTE_ETH_FOREACH_DEV(pi) {
1938                 if (!port_is_stopped(pi))
1939                         return 0;
1940         }
1941
1942         return 1;
1943 }
1944
1945 int
1946 port_is_started(portid_t port_id)
1947 {
1948         if (port_id_is_invalid(port_id, ENABLED_WARN))
1949                 return 0;
1950
1951         if (ports[port_id].port_status != RTE_PORT_STARTED)
1952                 return 0;
1953
1954         return 1;
1955 }
1956
1957 int
1958 start_port(portid_t pid)
1959 {
1960         int diag, need_check_link_status = -1;
1961         portid_t pi;
1962         queueid_t qi;
1963         struct rte_port *port;
1964         struct ether_addr mac_addr;
1965
1966         if (port_id_is_invalid(pid, ENABLED_WARN))
1967                 return 0;
1968
1969         if(dcb_config)
1970                 dcb_test = 1;
1971         RTE_ETH_FOREACH_DEV(pi) {
1972                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1973                         continue;
1974
1975                 need_check_link_status = 0;
1976                 port = &ports[pi];
1977                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1978                                                  RTE_PORT_HANDLING) == 0) {
1979                         printf("Port %d is now not stopped\n", pi);
1980                         continue;
1981                 }
1982
1983                 if (port->need_reconfig > 0) {
1984                         port->need_reconfig = 0;
1985
1986                         if (flow_isolate_all) {
1987                                 int ret = port_flow_isolate(pi, 1);
1988                                 if (ret) {
1989                                         printf("Failed to apply isolated"
1990                                                " mode on port %d\n", pi);
1991                                         return -1;
1992                                 }
1993                         }
1994                         configure_rxtx_dump_callbacks(0);
1995                         printf("Configuring Port %d (socket %u)\n", pi,
1996                                         port->socket_id);
1997                         /* configure port */
1998                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1999                                                 &(port->dev_conf));
2000                         if (diag != 0) {
2001                                 if (rte_atomic16_cmpset(&(port->port_status),
2002                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2003                                         printf("Port %d can not be set back "
2004                                                         "to stopped\n", pi);
2005                                 printf("Fail to configure port %d\n", pi);
2006                                 /* try to reconfigure port next time */
2007                                 port->need_reconfig = 1;
2008                                 return -1;
2009                         }
2010                 }
2011                 if (port->need_reconfig_queues > 0) {
2012                         port->need_reconfig_queues = 0;
2013                         /* setup tx queues */
2014                         for (qi = 0; qi < nb_txq; qi++) {
2015                                 if ((numa_support) &&
2016                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2017                                         diag = rte_eth_tx_queue_setup(pi, qi,
2018                                                 port->nb_tx_desc[qi],
2019                                                 txring_numa[pi],
2020                                                 &(port->tx_conf[qi]));
2021                                 else
2022                                         diag = rte_eth_tx_queue_setup(pi, qi,
2023                                                 port->nb_tx_desc[qi],
2024                                                 port->socket_id,
2025                                                 &(port->tx_conf[qi]));
2026
2027                                 if (diag == 0)
2028                                         continue;
2029
2030                                 /* Fail to setup tx queue, return */
2031                                 if (rte_atomic16_cmpset(&(port->port_status),
2032                                                         RTE_PORT_HANDLING,
2033                                                         RTE_PORT_STOPPED) == 0)
2034                                         printf("Port %d can not be set back "
2035                                                         "to stopped\n", pi);
2036                                 printf("Fail to configure port %d tx queues\n",
2037                                        pi);
2038                                 /* try to reconfigure queues next time */
2039                                 port->need_reconfig_queues = 1;
2040                                 return -1;
2041                         }
2042                         for (qi = 0; qi < nb_rxq; qi++) {
2043                                 /* setup rx queues */
2044                                 if ((numa_support) &&
2045                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2046                                         struct rte_mempool * mp =
2047                                                 mbuf_pool_find(rxring_numa[pi]);
2048                                         if (mp == NULL) {
2049                                                 printf("Failed to setup RX queue:"
2050                                                         "No mempool allocation"
2051                                                         " on the socket %d\n",
2052                                                         rxring_numa[pi]);
2053                                                 return -1;
2054                                         }
2055
2056                                         diag = rte_eth_rx_queue_setup(pi, qi,
2057                                              port->nb_rx_desc[qi],
2058                                              rxring_numa[pi],
2059                                              &(port->rx_conf[qi]),
2060                                              mp);
2061                                 } else {
2062                                         struct rte_mempool *mp =
2063                                                 mbuf_pool_find(port->socket_id);
2064                                         if (mp == NULL) {
2065                                                 printf("Failed to setup RX queue:"
2066                                                         "No mempool allocation"
2067                                                         " on the socket %d\n",
2068                                                         port->socket_id);
2069                                                 return -1;
2070                                         }
2071                                         diag = rte_eth_rx_queue_setup(pi, qi,
2072                                              port->nb_rx_desc[qi],
2073                                              port->socket_id,
2074                                              &(port->rx_conf[qi]),
2075                                              mp);
2076                                 }
2077                                 if (diag == 0)
2078                                         continue;
2079
2080                                 /* Fail to setup rx queue, return */
2081                                 if (rte_atomic16_cmpset(&(port->port_status),
2082                                                         RTE_PORT_HANDLING,
2083                                                         RTE_PORT_STOPPED) == 0)
2084                                         printf("Port %d can not be set back "
2085                                                         "to stopped\n", pi);
2086                                 printf("Fail to configure port %d rx queues\n",
2087                                        pi);
2088                                 /* try to reconfigure queues next time */
2089                                 port->need_reconfig_queues = 1;
2090                                 return -1;
2091                         }
2092                 }
2093                 configure_rxtx_dump_callbacks(verbose_level);
2094                 /* start port */
2095                 if (rte_eth_dev_start(pi) < 0) {
2096                         printf("Fail to start port %d\n", pi);
2097
2098                         /* Fail to setup rx queue, return */
2099                         if (rte_atomic16_cmpset(&(port->port_status),
2100                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2101                                 printf("Port %d can not be set back to "
2102                                                         "stopped\n", pi);
2103                         continue;
2104                 }
2105
2106                 if (rte_atomic16_cmpset(&(port->port_status),
2107                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2108                         printf("Port %d can not be set into started\n", pi);
2109
2110                 rte_eth_macaddr_get(pi, &mac_addr);
2111                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2112                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2113                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2114                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2115
2116                 /* at least one port started, need checking link status */
2117                 need_check_link_status = 1;
2118         }
2119
2120         if (need_check_link_status == 1 && !no_link_check)
2121                 check_all_ports_link_status(RTE_PORT_ALL);
2122         else if (need_check_link_status == 0)
2123                 printf("Please stop the ports first\n");
2124
2125         printf("Done\n");
2126         return 0;
2127 }
2128
2129 void
2130 stop_port(portid_t pid)
2131 {
2132         portid_t pi;
2133         struct rte_port *port;
2134         int need_check_link_status = 0;
2135
2136         if (dcb_test) {
2137                 dcb_test = 0;
2138                 dcb_config = 0;
2139         }
2140
2141         if (port_id_is_invalid(pid, ENABLED_WARN))
2142                 return;
2143
2144         printf("Stopping ports...\n");
2145
2146         RTE_ETH_FOREACH_DEV(pi) {
2147                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2148                         continue;
2149
2150                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2151                         printf("Please remove port %d from forwarding configuration.\n", pi);
2152                         continue;
2153                 }
2154
2155                 if (port_is_bonding_slave(pi)) {
2156                         printf("Please remove port %d from bonded device.\n", pi);
2157                         continue;
2158                 }
2159
2160                 port = &ports[pi];
2161                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2162                                                 RTE_PORT_HANDLING) == 0)
2163                         continue;
2164
2165                 rte_eth_dev_stop(pi);
2166
2167                 if (rte_atomic16_cmpset(&(port->port_status),
2168                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2169                         printf("Port %d can not be set into stopped\n", pi);
2170                 need_check_link_status = 1;
2171         }
2172         if (need_check_link_status && !no_link_check)
2173                 check_all_ports_link_status(RTE_PORT_ALL);
2174
2175         printf("Done\n");
2176 }
2177
2178 static void
2179 remove_invalid_ports_in(portid_t *array, portid_t *total)
2180 {
2181         portid_t i;
2182         portid_t new_total = 0;
2183
2184         for (i = 0; i < *total; i++)
2185                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2186                         array[new_total] = array[i];
2187                         new_total++;
2188                 }
2189         *total = new_total;
2190 }
2191
2192 static void
2193 remove_invalid_ports(void)
2194 {
2195         remove_invalid_ports_in(ports_ids, &nb_ports);
2196         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2197         nb_cfg_ports = nb_fwd_ports;
2198 }
2199
2200 void
2201 close_port(portid_t pid)
2202 {
2203         portid_t pi;
2204         struct rte_port *port;
2205
2206         if (port_id_is_invalid(pid, ENABLED_WARN))
2207                 return;
2208
2209         printf("Closing ports...\n");
2210
2211         RTE_ETH_FOREACH_DEV(pi) {
2212                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2213                         continue;
2214
2215                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2216                         printf("Please remove port %d from forwarding configuration.\n", pi);
2217                         continue;
2218                 }
2219
2220                 if (port_is_bonding_slave(pi)) {
2221                         printf("Please remove port %d from bonded device.\n", pi);
2222                         continue;
2223                 }
2224
2225                 port = &ports[pi];
2226                 if (rte_atomic16_cmpset(&(port->port_status),
2227                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2228                         printf("Port %d is already closed\n", pi);
2229                         continue;
2230                 }
2231
2232                 if (rte_atomic16_cmpset(&(port->port_status),
2233                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2234                         printf("Port %d is now not stopped\n", pi);
2235                         continue;
2236                 }
2237
2238                 if (port->flow_list)
2239                         port_flow_flush(pi);
2240                 rte_eth_dev_close(pi);
2241
2242                 remove_invalid_ports();
2243
2244                 if (rte_atomic16_cmpset(&(port->port_status),
2245                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2246                         printf("Port %d cannot be set to closed\n", pi);
2247         }
2248
2249         printf("Done\n");
2250 }
2251
2252 void
2253 reset_port(portid_t pid)
2254 {
2255         int diag;
2256         portid_t pi;
2257         struct rte_port *port;
2258
2259         if (port_id_is_invalid(pid, ENABLED_WARN))
2260                 return;
2261
2262         printf("Resetting ports...\n");
2263
2264         RTE_ETH_FOREACH_DEV(pi) {
2265                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2266                         continue;
2267
2268                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2269                         printf("Please remove port %d from forwarding "
2270                                "configuration.\n", pi);
2271                         continue;
2272                 }
2273
2274                 if (port_is_bonding_slave(pi)) {
2275                         printf("Please remove port %d from bonded device.\n",
2276                                pi);
2277                         continue;
2278                 }
2279
2280                 diag = rte_eth_dev_reset(pi);
2281                 if (diag == 0) {
2282                         port = &ports[pi];
2283                         port->need_reconfig = 1;
2284                         port->need_reconfig_queues = 1;
2285                 } else {
2286                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2287                 }
2288         }
2289
2290         printf("Done\n");
2291 }
2292
2293 void
2294 attach_port(char *identifier)
2295 {
2296         portid_t pi;
2297         struct rte_dev_iterator iterator;
2298
2299         printf("Attaching a new port...\n");
2300
2301         if (identifier == NULL) {
2302                 printf("Invalid parameters are specified\n");
2303                 return;
2304         }
2305
2306         if (rte_dev_probe(identifier) != 0) {
2307                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2308                 return;
2309         }
2310
2311         /* first attach mode: event */
2312         if (setup_on_probe_event) {
2313                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2314                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2315                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2316                                         ports[pi].need_setup != 0)
2317                                 setup_attached_port(pi);
2318                 return;
2319         }
2320
2321         /* second attach mode: iterator */
2322         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2323                 /* setup ports matching the devargs used for probing */
2324                 if (port_is_forwarding(pi))
2325                         continue; /* port was already attached before */
2326                 setup_attached_port(pi);
2327         }
2328 }
2329
2330 static void
2331 setup_attached_port(portid_t pi)
2332 {
2333         unsigned int socket_id;
2334
2335         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2336         /* if socket_id is invalid, set to the first available socket. */
2337         if (check_socket_id(socket_id) < 0)
2338                 socket_id = socket_ids[0];
2339         reconfig(pi, socket_id);
2340         rte_eth_promiscuous_enable(pi);
2341
2342         ports_ids[nb_ports++] = pi;
2343         fwd_ports_ids[nb_fwd_ports++] = pi;
2344         nb_cfg_ports = nb_fwd_ports;
2345         ports[pi].need_setup = 0;
2346         ports[pi].port_status = RTE_PORT_STOPPED;
2347
2348         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2349         printf("Done\n");
2350 }
2351
2352 void
2353 detach_port_device(portid_t port_id)
2354 {
2355         struct rte_device *dev;
2356         portid_t sibling;
2357
2358         printf("Removing a device...\n");
2359
2360         dev = rte_eth_devices[port_id].device;
2361         if (dev == NULL) {
2362                 printf("Device already removed\n");
2363                 return;
2364         }
2365
2366         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2367                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2368                         printf("Port not stopped\n");
2369                         return;
2370                 }
2371                 printf("Port was not closed\n");
2372                 if (ports[port_id].flow_list)
2373                         port_flow_flush(port_id);
2374         }
2375
2376         if (rte_dev_remove(dev) != 0) {
2377                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2378                 return;
2379         }
2380
2381         RTE_ETH_FOREACH_DEV_SIBLING(sibling, port_id) {
2382                 /* reset mapping between old ports and removed device */
2383                 rte_eth_devices[sibling].device = NULL;
2384                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2385                         /* sibling ports are forced to be closed */
2386                         ports[sibling].port_status = RTE_PORT_CLOSED;
2387                         printf("Port %u is closed\n", sibling);
2388                 }
2389         }
2390
2391         remove_invalid_ports();
2392
2393         printf("Device of port %u is detached\n", port_id);
2394         printf("Now total ports is %d\n", nb_ports);
2395         printf("Done\n");
2396         return;
2397 }
2398
2399 void
2400 pmd_test_exit(void)
2401 {
2402         struct rte_device *device;
2403         portid_t pt_id;
2404         int ret;
2405         int i;
2406
2407         if (test_done == 0)
2408                 stop_packet_forwarding();
2409
2410         if (ports != NULL) {
2411                 no_link_check = 1;
2412                 RTE_ETH_FOREACH_DEV(pt_id) {
2413                         printf("\nStopping port %d...\n", pt_id);
2414                         fflush(stdout);
2415                         stop_port(pt_id);
2416                 }
2417                 RTE_ETH_FOREACH_DEV(pt_id) {
2418                         printf("\nShutting down port %d...\n", pt_id);
2419                         fflush(stdout);
2420                         close_port(pt_id);
2421
2422                         /*
2423                          * This is a workaround to fix a virtio-user issue that
2424                          * requires to call clean-up routine to remove existing
2425                          * socket.
2426                          * This workaround valid only for testpmd, needs a fix
2427                          * valid for all applications.
2428                          * TODO: Implement proper resource cleanup
2429                          */
2430                         device = rte_eth_devices[pt_id].device;
2431                         if (device && !strcmp(device->driver->name, "net_virtio_user"))
2432                                 detach_port_device(pt_id);
2433                 }
2434         }
2435
2436         if (hot_plug) {
2437                 ret = rte_dev_event_monitor_stop();
2438                 if (ret) {
2439                         RTE_LOG(ERR, EAL,
2440                                 "fail to stop device event monitor.");
2441                         return;
2442                 }
2443
2444                 ret = rte_dev_event_callback_unregister(NULL,
2445                         dev_event_callback, NULL);
2446                 if (ret < 0) {
2447                         RTE_LOG(ERR, EAL,
2448                                 "fail to unregister device event callback.\n");
2449                         return;
2450                 }
2451
2452                 ret = rte_dev_hotplug_handle_disable();
2453                 if (ret) {
2454                         RTE_LOG(ERR, EAL,
2455                                 "fail to disable hotplug handling.\n");
2456                         return;
2457                 }
2458         }
2459         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2460                 if (mempools[i])
2461                         rte_mempool_free(mempools[i]);
2462         }
2463
2464         printf("\nBye...\n");
2465 }
2466
2467 typedef void (*cmd_func_t)(void);
2468 struct pmd_test_command {
2469         const char *cmd_name;
2470         cmd_func_t cmd_func;
2471 };
2472
2473 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2474
2475 /* Check the link status of all ports in up to 9s, and print them finally */
2476 static void
2477 check_all_ports_link_status(uint32_t port_mask)
2478 {
2479 #define CHECK_INTERVAL 100 /* 100ms */
2480 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2481         portid_t portid;
2482         uint8_t count, all_ports_up, print_flag = 0;
2483         struct rte_eth_link link;
2484
2485         printf("Checking link statuses...\n");
2486         fflush(stdout);
2487         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2488                 all_ports_up = 1;
2489                 RTE_ETH_FOREACH_DEV(portid) {
2490                         if ((port_mask & (1 << portid)) == 0)
2491                                 continue;
2492                         memset(&link, 0, sizeof(link));
2493                         rte_eth_link_get_nowait(portid, &link);
2494                         /* print link status if flag set */
2495                         if (print_flag == 1) {
2496                                 if (link.link_status)
2497                                         printf(
2498                                         "Port%d Link Up. speed %u Mbps- %s\n",
2499                                         portid, link.link_speed,
2500                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2501                                         ("full-duplex") : ("half-duplex\n"));
2502                                 else
2503                                         printf("Port %d Link Down\n", portid);
2504                                 continue;
2505                         }
2506                         /* clear all_ports_up flag if any link down */
2507                         if (link.link_status == ETH_LINK_DOWN) {
2508                                 all_ports_up = 0;
2509                                 break;
2510                         }
2511                 }
2512                 /* after finally printing all link status, get out */
2513                 if (print_flag == 1)
2514                         break;
2515
2516                 if (all_ports_up == 0) {
2517                         fflush(stdout);
2518                         rte_delay_ms(CHECK_INTERVAL);
2519                 }
2520
2521                 /* set the print_flag if all ports up or timeout */
2522                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2523                         print_flag = 1;
2524                 }
2525
2526                 if (lsc_interrupt)
2527                         break;
2528         }
2529 }
2530
2531 /*
2532  * This callback is for remove a port for a device. It has limitation because
2533  * it is not for multiple port removal for a device.
2534  * TODO: the device detach invoke will plan to be removed from user side to
2535  * eal. And convert all PMDs to free port resources on ether device closing.
2536  */
2537 static void
2538 rmv_port_callback(void *arg)
2539 {
2540         int need_to_start = 0;
2541         int org_no_link_check = no_link_check;
2542         portid_t port_id = (intptr_t)arg;
2543
2544         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2545
2546         if (!test_done && port_is_forwarding(port_id)) {
2547                 need_to_start = 1;
2548                 stop_packet_forwarding();
2549         }
2550         no_link_check = 1;
2551         stop_port(port_id);
2552         no_link_check = org_no_link_check;
2553         close_port(port_id);
2554         detach_port_device(port_id);
2555         if (need_to_start)
2556                 start_packet_forwarding(0);
2557 }
2558
2559 /* This function is used by the interrupt thread */
2560 static int
2561 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2562                   void *ret_param)
2563 {
2564         RTE_SET_USED(param);
2565         RTE_SET_USED(ret_param);
2566
2567         if (type >= RTE_ETH_EVENT_MAX) {
2568                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2569                         port_id, __func__, type);
2570                 fflush(stderr);
2571         } else if (event_print_mask & (UINT32_C(1) << type)) {
2572                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2573                         eth_event_desc[type]);
2574                 fflush(stdout);
2575         }
2576
2577         switch (type) {
2578         case RTE_ETH_EVENT_NEW:
2579                 ports[port_id].need_setup = 1;
2580                 ports[port_id].port_status = RTE_PORT_HANDLING;
2581                 break;
2582         case RTE_ETH_EVENT_INTR_RMV:
2583                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2584                         break;
2585                 if (rte_eal_alarm_set(100000,
2586                                 rmv_port_callback, (void *)(intptr_t)port_id))
2587                         fprintf(stderr, "Could not set up deferred device removal\n");
2588                 break;
2589         default:
2590                 break;
2591         }
2592         return 0;
2593 }
2594
2595 static int
2596 register_eth_event_callback(void)
2597 {
2598         int ret;
2599         enum rte_eth_event_type event;
2600
2601         for (event = RTE_ETH_EVENT_UNKNOWN;
2602                         event < RTE_ETH_EVENT_MAX; event++) {
2603                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2604                                 event,
2605                                 eth_event_callback,
2606                                 NULL);
2607                 if (ret != 0) {
2608                         TESTPMD_LOG(ERR, "Failed to register callback for "
2609                                         "%s event\n", eth_event_desc[event]);
2610                         return -1;
2611                 }
2612         }
2613
2614         return 0;
2615 }
2616
2617 /* This function is used by the interrupt thread */
2618 static void
2619 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2620                              __rte_unused void *arg)
2621 {
2622         uint16_t port_id;
2623         int ret;
2624
2625         if (type >= RTE_DEV_EVENT_MAX) {
2626                 fprintf(stderr, "%s called upon invalid event %d\n",
2627                         __func__, type);
2628                 fflush(stderr);
2629         }
2630
2631         switch (type) {
2632         case RTE_DEV_EVENT_REMOVE:
2633                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2634                         device_name);
2635                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2636                 if (ret) {
2637                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2638                                 device_name);
2639                         return;
2640                 }
2641                 /*
2642                  * Because the user's callback is invoked in eal interrupt
2643                  * callback, the interrupt callback need to be finished before
2644                  * it can be unregistered when detaching device. So finish
2645                  * callback soon and use a deferred removal to detach device
2646                  * is need. It is a workaround, once the device detaching be
2647                  * moved into the eal in the future, the deferred removal could
2648                  * be deleted.
2649                  */
2650                 if (rte_eal_alarm_set(100000,
2651                                 rmv_port_callback, (void *)(intptr_t)port_id))
2652                         RTE_LOG(ERR, EAL,
2653                                 "Could not set up deferred device removal\n");
2654                 break;
2655         case RTE_DEV_EVENT_ADD:
2656                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2657                         device_name);
2658                 /* TODO: After finish kernel driver binding,
2659                  * begin to attach port.
2660                  */
2661                 break;
2662         default:
2663                 break;
2664         }
2665 }
2666
2667 static int
2668 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2669 {
2670         uint16_t i;
2671         int diag;
2672         uint8_t mapping_found = 0;
2673
2674         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2675                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2676                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2677                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2678                                         tx_queue_stats_mappings[i].queue_id,
2679                                         tx_queue_stats_mappings[i].stats_counter_id);
2680                         if (diag != 0)
2681                                 return diag;
2682                         mapping_found = 1;
2683                 }
2684         }
2685         if (mapping_found)
2686                 port->tx_queue_stats_mapping_enabled = 1;
2687         return 0;
2688 }
2689
2690 static int
2691 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2692 {
2693         uint16_t i;
2694         int diag;
2695         uint8_t mapping_found = 0;
2696
2697         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2698                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2699                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2700                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2701                                         rx_queue_stats_mappings[i].queue_id,
2702                                         rx_queue_stats_mappings[i].stats_counter_id);
2703                         if (diag != 0)
2704                                 return diag;
2705                         mapping_found = 1;
2706                 }
2707         }
2708         if (mapping_found)
2709                 port->rx_queue_stats_mapping_enabled = 1;
2710         return 0;
2711 }
2712
2713 static void
2714 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2715 {
2716         int diag = 0;
2717
2718         diag = set_tx_queue_stats_mapping_registers(pi, port);
2719         if (diag != 0) {
2720                 if (diag == -ENOTSUP) {
2721                         port->tx_queue_stats_mapping_enabled = 0;
2722                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2723                 }
2724                 else
2725                         rte_exit(EXIT_FAILURE,
2726                                         "set_tx_queue_stats_mapping_registers "
2727                                         "failed for port id=%d diag=%d\n",
2728                                         pi, diag);
2729         }
2730
2731         diag = set_rx_queue_stats_mapping_registers(pi, port);
2732         if (diag != 0) {
2733                 if (diag == -ENOTSUP) {
2734                         port->rx_queue_stats_mapping_enabled = 0;
2735                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2736                 }
2737                 else
2738                         rte_exit(EXIT_FAILURE,
2739                                         "set_rx_queue_stats_mapping_registers "
2740                                         "failed for port id=%d diag=%d\n",
2741                                         pi, diag);
2742         }
2743 }
2744
2745 static void
2746 rxtx_port_config(struct rte_port *port)
2747 {
2748         uint16_t qid;
2749
2750         for (qid = 0; qid < nb_rxq; qid++) {
2751                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2752
2753                 /* Check if any Rx parameters have been passed */
2754                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2755                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2756
2757                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2758                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2759
2760                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2761                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2762
2763                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2764                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2765
2766                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2767                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2768
2769                 port->nb_rx_desc[qid] = nb_rxd;
2770         }
2771
2772         for (qid = 0; qid < nb_txq; qid++) {
2773                 port->tx_conf[qid] = port->dev_info.default_txconf;
2774
2775                 /* Check if any Tx parameters have been passed */
2776                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2777                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2778
2779                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2780                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2781
2782                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2783                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2784
2785                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2786                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2787
2788                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2789                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2790
2791                 port->nb_tx_desc[qid] = nb_txd;
2792         }
2793 }
2794
2795 void
2796 init_port_config(void)
2797 {
2798         portid_t pid;
2799         struct rte_port *port;
2800
2801         RTE_ETH_FOREACH_DEV(pid) {
2802                 port = &ports[pid];
2803                 port->dev_conf.fdir_conf = fdir_conf;
2804                 rte_eth_dev_info_get(pid, &port->dev_info);
2805                 if (nb_rxq > 1) {
2806                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2807                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2808                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2809                 } else {
2810                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2811                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2812                 }
2813
2814                 if (port->dcb_flag == 0) {
2815                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2816                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2817                         else
2818                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2819                 }
2820
2821                 rxtx_port_config(port);
2822
2823                 rte_eth_macaddr_get(pid, &port->eth_addr);
2824
2825                 map_port_queue_stats_mapping_registers(pid, port);
2826 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2827                 rte_pmd_ixgbe_bypass_init(pid);
2828 #endif
2829
2830                 if (lsc_interrupt &&
2831                     (rte_eth_devices[pid].data->dev_flags &
2832                      RTE_ETH_DEV_INTR_LSC))
2833                         port->dev_conf.intr_conf.lsc = 1;
2834                 if (rmv_interrupt &&
2835                     (rte_eth_devices[pid].data->dev_flags &
2836                      RTE_ETH_DEV_INTR_RMV))
2837                         port->dev_conf.intr_conf.rmv = 1;
2838         }
2839 }
2840
2841 void set_port_slave_flag(portid_t slave_pid)
2842 {
2843         struct rte_port *port;
2844
2845         port = &ports[slave_pid];
2846         port->slave_flag = 1;
2847 }
2848
2849 void clear_port_slave_flag(portid_t slave_pid)
2850 {
2851         struct rte_port *port;
2852
2853         port = &ports[slave_pid];
2854         port->slave_flag = 0;
2855 }
2856
2857 uint8_t port_is_bonding_slave(portid_t slave_pid)
2858 {
2859         struct rte_port *port;
2860
2861         port = &ports[slave_pid];
2862         if ((rte_eth_devices[slave_pid].data->dev_flags &
2863             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
2864                 return 1;
2865         return 0;
2866 }
2867
2868 const uint16_t vlan_tags[] = {
2869                 0,  1,  2,  3,  4,  5,  6,  7,
2870                 8,  9, 10, 11,  12, 13, 14, 15,
2871                 16, 17, 18, 19, 20, 21, 22, 23,
2872                 24, 25, 26, 27, 28, 29, 30, 31
2873 };
2874
2875 static  int
2876 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
2877                  enum dcb_mode_enable dcb_mode,
2878                  enum rte_eth_nb_tcs num_tcs,
2879                  uint8_t pfc_en)
2880 {
2881         uint8_t i;
2882         int32_t rc;
2883         struct rte_eth_rss_conf rss_conf;
2884
2885         /*
2886          * Builds up the correct configuration for dcb+vt based on the vlan tags array
2887          * given above, and the number of traffic classes available for use.
2888          */
2889         if (dcb_mode == DCB_VT_ENABLED) {
2890                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
2891                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
2892                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2893                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
2894
2895                 /* VMDQ+DCB RX and TX configurations */
2896                 vmdq_rx_conf->enable_default_pool = 0;
2897                 vmdq_rx_conf->default_pool = 0;
2898                 vmdq_rx_conf->nb_queue_pools =
2899                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2900                 vmdq_tx_conf->nb_queue_pools =
2901                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2902
2903                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
2904                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
2905                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
2906                         vmdq_rx_conf->pool_map[i].pools =
2907                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
2908                 }
2909                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2910                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
2911                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
2912                 }
2913
2914                 /* set DCB mode of RX and TX of multiple queues */
2915                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
2916                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
2917         } else {
2918                 struct rte_eth_dcb_rx_conf *rx_conf =
2919                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
2920                 struct rte_eth_dcb_tx_conf *tx_conf =
2921                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
2922
2923                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
2924                 if (rc != 0)
2925                         return rc;
2926
2927                 rx_conf->nb_tcs = num_tcs;
2928                 tx_conf->nb_tcs = num_tcs;
2929
2930                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2931                         rx_conf->dcb_tc[i] = i % num_tcs;
2932                         tx_conf->dcb_tc[i] = i % num_tcs;
2933                 }
2934
2935                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
2936                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
2937                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
2938         }
2939
2940         if (pfc_en)
2941                 eth_conf->dcb_capability_en =
2942                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
2943         else
2944                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
2945
2946         return 0;
2947 }
2948
2949 int
2950 init_port_dcb_config(portid_t pid,
2951                      enum dcb_mode_enable dcb_mode,
2952                      enum rte_eth_nb_tcs num_tcs,
2953                      uint8_t pfc_en)
2954 {
2955         struct rte_eth_conf port_conf;
2956         struct rte_port *rte_port;
2957         int retval;
2958         uint16_t i;
2959
2960         rte_port = &ports[pid];
2961
2962         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
2963         /* Enter DCB configuration status */
2964         dcb_config = 1;
2965
2966         port_conf.rxmode = rte_port->dev_conf.rxmode;
2967         port_conf.txmode = rte_port->dev_conf.txmode;
2968
2969         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
2970         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
2971         if (retval < 0)
2972                 return retval;
2973         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
2974
2975         /* re-configure the device . */
2976         rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
2977
2978         rte_eth_dev_info_get(pid, &rte_port->dev_info);
2979
2980         /* If dev_info.vmdq_pool_base is greater than 0,
2981          * the queue id of vmdq pools is started after pf queues.
2982          */
2983         if (dcb_mode == DCB_VT_ENABLED &&
2984             rte_port->dev_info.vmdq_pool_base > 0) {
2985                 printf("VMDQ_DCB multi-queue mode is nonsensical"
2986                         " for port %d.", pid);
2987                 return -1;
2988         }
2989
2990         /* Assume the ports in testpmd have the same dcb capability
2991          * and has the same number of rxq and txq in dcb mode
2992          */
2993         if (dcb_mode == DCB_VT_ENABLED) {
2994                 if (rte_port->dev_info.max_vfs > 0) {
2995                         nb_rxq = rte_port->dev_info.nb_rx_queues;
2996                         nb_txq = rte_port->dev_info.nb_tx_queues;
2997                 } else {
2998                         nb_rxq = rte_port->dev_info.max_rx_queues;
2999                         nb_txq = rte_port->dev_info.max_tx_queues;
3000                 }
3001         } else {
3002                 /*if vt is disabled, use all pf queues */
3003                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3004                         nb_rxq = rte_port->dev_info.max_rx_queues;
3005                         nb_txq = rte_port->dev_info.max_tx_queues;
3006                 } else {
3007                         nb_rxq = (queueid_t)num_tcs;
3008                         nb_txq = (queueid_t)num_tcs;
3009
3010                 }
3011         }
3012         rx_free_thresh = 64;
3013
3014         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3015
3016         rxtx_port_config(rte_port);
3017         /* VLAN filter */
3018         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3019         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3020                 rx_vft_set(pid, vlan_tags[i], 1);
3021
3022         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
3023         map_port_queue_stats_mapping_registers(pid, rte_port);
3024
3025         rte_port->dcb_flag = 1;
3026
3027         return 0;
3028 }
3029
3030 static void
3031 init_port(void)
3032 {
3033         /* Configuration of Ethernet ports. */
3034         ports = rte_zmalloc("testpmd: ports",
3035                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3036                             RTE_CACHE_LINE_SIZE);
3037         if (ports == NULL) {
3038                 rte_exit(EXIT_FAILURE,
3039                                 "rte_zmalloc(%d struct rte_port) failed\n",
3040                                 RTE_MAX_ETHPORTS);
3041         }
3042
3043         /* Initialize ports NUMA structures */
3044         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3045         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3046         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3047 }
3048
3049 static void
3050 force_quit(void)
3051 {
3052         pmd_test_exit();
3053         prompt_exit();
3054 }
3055
3056 static void
3057 print_stats(void)
3058 {
3059         uint8_t i;
3060         const char clr[] = { 27, '[', '2', 'J', '\0' };
3061         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3062
3063         /* Clear screen and move to top left */
3064         printf("%s%s", clr, top_left);
3065
3066         printf("\nPort statistics ====================================");
3067         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3068                 nic_stats_display(fwd_ports_ids[i]);
3069
3070         fflush(stdout);
3071 }
3072
3073 static void
3074 signal_handler(int signum)
3075 {
3076         if (signum == SIGINT || signum == SIGTERM) {
3077                 printf("\nSignal %d received, preparing to exit...\n",
3078                                 signum);
3079 #ifdef RTE_LIBRTE_PDUMP
3080                 /* uninitialize packet capture framework */
3081                 rte_pdump_uninit();
3082 #endif
3083 #ifdef RTE_LIBRTE_LATENCY_STATS
3084                 rte_latencystats_uninit();
3085 #endif
3086                 force_quit();
3087                 /* Set flag to indicate the force termination. */
3088                 f_quit = 1;
3089                 /* exit with the expected status */
3090                 signal(signum, SIG_DFL);
3091                 kill(getpid(), signum);
3092         }
3093 }
3094
3095 int
3096 main(int argc, char** argv)
3097 {
3098         int diag;
3099         portid_t port_id;
3100         uint16_t count;
3101         int ret;
3102
3103         signal(SIGINT, signal_handler);
3104         signal(SIGTERM, signal_handler);
3105
3106         diag = rte_eal_init(argc, argv);
3107         if (diag < 0)
3108                 rte_panic("Cannot init EAL\n");
3109
3110         testpmd_logtype = rte_log_register("testpmd");
3111         if (testpmd_logtype < 0)
3112                 rte_panic("Cannot register log type");
3113         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3114
3115         ret = register_eth_event_callback();
3116         if (ret != 0)
3117                 rte_panic("Cannot register for ethdev events");
3118
3119 #ifdef RTE_LIBRTE_PDUMP
3120         /* initialize packet capture framework */
3121         rte_pdump_init();
3122 #endif
3123
3124         count = 0;
3125         RTE_ETH_FOREACH_DEV(port_id) {
3126                 ports_ids[count] = port_id;
3127                 count++;
3128         }
3129         nb_ports = (portid_t) count;
3130         if (nb_ports == 0)
3131                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3132
3133         /* allocate port structures, and init them */
3134         init_port();
3135
3136         set_def_fwd_config();
3137         if (nb_lcores == 0)
3138                 rte_panic("Empty set of forwarding logical cores - check the "
3139                           "core mask supplied in the command parameters\n");
3140
3141         /* Bitrate/latency stats disabled by default */
3142 #ifdef RTE_LIBRTE_BITRATE
3143         bitrate_enabled = 0;
3144 #endif
3145 #ifdef RTE_LIBRTE_LATENCY_STATS
3146         latencystats_enabled = 0;
3147 #endif
3148
3149         /* on FreeBSD, mlockall() is disabled by default */
3150 #ifdef RTE_EXEC_ENV_FREEBSD
3151         do_mlockall = 0;
3152 #else
3153         do_mlockall = 1;
3154 #endif
3155
3156         argc -= diag;
3157         argv += diag;
3158         if (argc > 1)
3159                 launch_args_parse(argc, argv);
3160
3161         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3162                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3163                         strerror(errno));
3164         }
3165
3166         if (tx_first && interactive)
3167                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3168                                 "interactive mode.\n");
3169
3170         if (tx_first && lsc_interrupt) {
3171                 printf("Warning: lsc_interrupt needs to be off when "
3172                                 " using tx_first. Disabling.\n");
3173                 lsc_interrupt = 0;
3174         }
3175
3176         if (!nb_rxq && !nb_txq)
3177                 printf("Warning: Either rx or tx queues should be non-zero\n");
3178
3179         if (nb_rxq > 1 && nb_rxq > nb_txq)
3180                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3181                        "but nb_txq=%d will prevent to fully test it.\n",
3182                        nb_rxq, nb_txq);
3183
3184         init_config();
3185
3186         if (hot_plug) {
3187                 ret = rte_dev_hotplug_handle_enable();
3188                 if (ret) {
3189                         RTE_LOG(ERR, EAL,
3190                                 "fail to enable hotplug handling.");
3191                         return -1;
3192                 }
3193
3194                 ret = rte_dev_event_monitor_start();
3195                 if (ret) {
3196                         RTE_LOG(ERR, EAL,
3197                                 "fail to start device event monitoring.");
3198                         return -1;
3199                 }
3200
3201                 ret = rte_dev_event_callback_register(NULL,
3202                         dev_event_callback, NULL);
3203                 if (ret) {
3204                         RTE_LOG(ERR, EAL,
3205                                 "fail  to register device event callback\n");
3206                         return -1;
3207                 }
3208         }
3209
3210         if (start_port(RTE_PORT_ALL) != 0)
3211                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3212
3213         /* set all ports to promiscuous mode by default */
3214         RTE_ETH_FOREACH_DEV(port_id)
3215                 rte_eth_promiscuous_enable(port_id);
3216
3217         /* Init metrics library */
3218         rte_metrics_init(rte_socket_id());
3219
3220 #ifdef RTE_LIBRTE_LATENCY_STATS
3221         if (latencystats_enabled != 0) {
3222                 int ret = rte_latencystats_init(1, NULL);
3223                 if (ret)
3224                         printf("Warning: latencystats init()"
3225                                 " returned error %d\n", ret);
3226                 printf("Latencystats running on lcore %d\n",
3227                         latencystats_lcore_id);
3228         }
3229 #endif
3230
3231         /* Setup bitrate stats */
3232 #ifdef RTE_LIBRTE_BITRATE
3233         if (bitrate_enabled != 0) {
3234                 bitrate_data = rte_stats_bitrate_create();
3235                 if (bitrate_data == NULL)
3236                         rte_exit(EXIT_FAILURE,
3237                                 "Could not allocate bitrate data.\n");
3238                 rte_stats_bitrate_reg(bitrate_data);
3239         }
3240 #endif
3241
3242 #ifdef RTE_LIBRTE_CMDLINE
3243         if (strlen(cmdline_filename) != 0)
3244                 cmdline_read_from_file(cmdline_filename);
3245
3246         if (interactive == 1) {
3247                 if (auto_start) {
3248                         printf("Start automatic packet forwarding\n");
3249                         start_packet_forwarding(0);
3250                 }
3251                 prompt();
3252                 pmd_test_exit();
3253         } else
3254 #endif
3255         {
3256                 char c;
3257                 int rc;
3258
3259                 f_quit = 0;
3260
3261                 printf("No commandline core given, start packet forwarding\n");
3262                 start_packet_forwarding(tx_first);
3263                 if (stats_period != 0) {
3264                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3265                         uint64_t timer_period;
3266
3267                         /* Convert to number of cycles */
3268                         timer_period = stats_period * rte_get_timer_hz();
3269
3270                         while (f_quit == 0) {
3271                                 cur_time = rte_get_timer_cycles();
3272                                 diff_time += cur_time - prev_time;
3273
3274                                 if (diff_time >= timer_period) {
3275                                         print_stats();
3276                                         /* Reset the timer */
3277                                         diff_time = 0;
3278                                 }
3279                                 /* Sleep to avoid unnecessary checks */
3280                                 prev_time = cur_time;
3281                                 sleep(1);
3282                         }
3283                 }
3284
3285                 printf("Press enter to exit\n");
3286                 rc = read(0, &c, 1);
3287                 pmd_test_exit();
3288                 if (rc < 0)
3289                         return 1;
3290         }
3291
3292         return 0;
3293 }