app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_malloc_heap.h>
  31 #include <rte_memory.h>
  32 #include <rte_memcpy.h>
  33 #include <rte_launch.h>
  34 #include <rte_eal.h>
  35 #include <rte_alarm.h>
  36 #include <rte_per_lcore.h>
  37 #include <rte_lcore.h>
  38 #include <rte_atomic.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_LIBRTE_IXGBE_PMD
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIBRTE_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #include <rte_metrics.h>
  58 #ifdef RTE_LIBRTE_BITRATE
  59 #include <rte_bitrate.h>
  60 #endif
  61 #ifdef RTE_LIBRTE_LATENCY_STATS
  62 #include <rte_latencystats.h>
  63 #endif
  64
  65 #include "testpmd.h"
  66
  67 #ifndef MAP_HUGETLB
  68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  69 #define HUGE_FLAG (0x40000)
  70 #else
  71 #define HUGE_FLAG MAP_HUGETLB
  72 #endif
  73
  74 #ifndef MAP_HUGE_SHIFT
  75 /* older kernels (or FreeBSD) will not have this define */
  76 #define HUGE_SHIFT (26)
  77 #else
  78 #define HUGE_SHIFT MAP_HUGE_SHIFT
  79 #endif
  80
  81 #define EXTMEM_HEAP_NAME "extmem"
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct fwd_config cur_fwd_config;
 192 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 193 uint32_t retry_enabled;
 194 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 195 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 196
 197 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 198 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 199                                       * specified on command-line. */
 200 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 201
 202 /*
 203  * In container, it cannot terminate the process which running with 'stats-period'
 204  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 205  */
 206 uint8_t f_quit;
 207
 208 /*
 209  * Configuration of packet segments used by the "txonly" processing engine.
 210  */
 211 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 212 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 213         TXONLY_DEF_PACKET_LEN,
 214 };
 215 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 216
 217 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 218 /**< Split policy for packets to TX. */
 219
 220 uint8_t txonly_multi_flow;
 221 /**< Whether multiple flows are generated in TXONLY mode. */
 222
 223 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 224 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 225
 226 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 227 uint8_t dcb_config = 0;
 228
 229 /* Whether the dcb is in testing status */
 230 uint8_t dcb_test = 0;
 231
 232 /*
 233  * Configurable number of RX/TX queues.
 234  */
 235 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 236 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 237
 238 /*
 239  * Configurable number of RX/TX ring descriptors.
 240  * Defaults are supplied by drivers via ethdev.
 241  */
 242 #define RTE_TEST_RX_DESC_DEFAULT 0
 243 #define RTE_TEST_TX_DESC_DEFAULT 0
 244 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 245 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 246
 247 #define RTE_PMD_PARAM_UNSET -1
 248 /*
 249  * Configurable values of RX and TX ring threshold registers.
 250  */
 251
 252 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 253 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 254 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 255
 256 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 258 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 259
 260 /*
 261  * Configurable value of RX free threshold.
 262  */
 263 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 264
 265 /*
 266  * Configurable value of RX drop enable.
 267  */
 268 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 269
 270 /*
 271  * Configurable value of TX free threshold.
 272  */
 273 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 274
 275 /*
 276  * Configurable value of TX RS bit threshold.
 277  */
 278 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 279
 280 /*
 281  * Configurable value of buffered packets before sending.
 282  */
 283 uint16_t noisy_tx_sw_bufsz;
 284
 285 /*
 286  * Configurable value of packet buffer timeout.
 287  */
 288 uint16_t noisy_tx_sw_buf_flush_time;
 289
 290 /*
 291  * Configurable value for size of VNF internal memory area
 292  * used for simulating noisy neighbour behaviour
 293  */
 294 uint64_t noisy_lkup_mem_sz;
 295
 296 /*
 297  * Configurable value of number of random writes done in
 298  * VNF simulation memory area.
 299  */
 300 uint64_t noisy_lkup_num_writes;
 301
 302 /*
 303  * Configurable value of number of random reads done in
 304  * VNF simulation memory area.
 305  */
 306 uint64_t noisy_lkup_num_reads;
 307
 308 /*
 309  * Configurable value of number of random reads/writes done in
 310  * VNF simulation memory area.
 311  */
 312 uint64_t noisy_lkup_num_reads_writes;
 313
 314 /*
 315  * Receive Side Scaling (RSS) configuration.
 316  */
 317 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 318
 319 /*
 320  * Port topology configuration
 321  */
 322 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 323
 324 /*
 325  * Avoids to flush all the RX streams before starts forwarding.
 326  */
 327 uint8_t no_flush_rx = 0; /* flush by default */
 328
 329 /*
 330  * Flow API isolated mode.
 331  */
 332 uint8_t flow_isolate_all;
 333
 334 /*
 335  * Avoids to check link status when starting/stopping a port.
 336  */
 337 uint8_t no_link_check = 0; /* check by default */
 338
 339 /*
 340  * Enable link status change notification
 341  */
 342 uint8_t lsc_interrupt = 1; /* enabled by default */
 343
 344 /*
 345  * Enable device removal notification.
 346  */
 347 uint8_t rmv_interrupt = 1; /* enabled by default */
 348
 349 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 350
 351 /* After attach, port setup is called on event or by iterator */
 352 bool setup_on_probe_event = true;
 353
 354 /* Pretty printing of ethdev events */
 355 static const char * const eth_event_desc[] = {
 356         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 357         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 358         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 359         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 360         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 361         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 362         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 363         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 364         [RTE_ETH_EVENT_NEW] = "device probed",
 365         [RTE_ETH_EVENT_DESTROY] = "device released",
 366         [RTE_ETH_EVENT_MAX] = NULL,
 367 };
 368
 369 /*
 370  * Display or mask ether events
 371  * Default to all events except VF_MBOX
 372  */
 373 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 374                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 375                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 376                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 377                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 378                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 379                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 380 /*
 381  * Decide if all memory are locked for performance.
 382  */
 383 int do_mlockall = 0;
 384
 385 /*
 386  * NIC bypass mode configuration options.
 387  */
 388
 389 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 390 /* The NIC bypass watchdog timeout. */
 391 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 392 #endif
 393
 394
 395 #ifdef RTE_LIBRTE_LATENCY_STATS
 396
 397 /*
 398  * Set when latency stats is enabled in the commandline
 399  */
 400 uint8_t latencystats_enabled;
 401
 402 /*
 403  * Lcore ID to serive latency statistics.
 404  */
 405 lcoreid_t latencystats_lcore_id = -1;
 406
 407 #endif
 408
 409 /*
 410  * Ethernet device configuration.
 411  */
 412 struct rte_eth_rxmode rx_mode = {
 413         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 414 };
 415
 416 struct rte_eth_txmode tx_mode = {
 417         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 418 };
 419
 420 struct rte_fdir_conf fdir_conf = {
 421         .mode = RTE_FDIR_MODE_NONE,
 422         .pballoc = RTE_FDIR_PBALLOC_64K,
 423         .status = RTE_FDIR_REPORT_STATUS,
 424         .mask = {
 425                 .vlan_tci_mask = 0xFFEF,
 426                 .ipv4_mask     = {
 427                         .src_ip = 0xFFFFFFFF,
 428                         .dst_ip = 0xFFFFFFFF,
 429                 },
 430                 .ipv6_mask     = {
 431                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 432                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 433                 },
 434                 .src_port_mask = 0xFFFF,
 435                 .dst_port_mask = 0xFFFF,
 436                 .mac_addr_byte_mask = 0xFF,
 437                 .tunnel_type_mask = 1,
 438                 .tunnel_id_mask = 0xFFFFFFFF,
 439         },
 440         .drop_queue = 127,
 441 };
 442
 443 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 444
 445 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 446 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 447
 448 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 449 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 450
 451 uint16_t nb_tx_queue_stats_mappings = 0;
 452 uint16_t nb_rx_queue_stats_mappings = 0;
 453
 454 /*
 455  * Display zero values by default for xstats
 456  */
 457 uint8_t xstats_hide_zero;
 458
 459 unsigned int num_sockets = 0;
 460 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 461
 462 #ifdef RTE_LIBRTE_BITRATE
 463 /* Bitrate statistics */
 464 struct rte_stats_bitrates *bitrate_data;
 465 lcoreid_t bitrate_lcore_id;
 466 uint8_t bitrate_enabled;
 467 #endif
 468
 469 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 470 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 471
 472 struct vxlan_encap_conf vxlan_encap_conf = {
 473         .select_ipv4 = 1,
 474         .select_vlan = 0,
 475         .select_tos_ttl = 0,
 476         .vni = "\x00\x00\x00",
 477         .udp_src = 0,
 478         .udp_dst = RTE_BE16(4789),
 479         .ipv4_src = IPv4(127, 0, 0, 1),
 480         .ipv4_dst = IPv4(255, 255, 255, 255),
 481         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 482                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 483         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 484                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 485         .vlan_tci = 0,
 486         .ip_tos = 0,
 487         .ip_ttl = 255,
 488         .eth_src = "\x00\x00\x00\x00\x00\x00",
 489         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 490 };
 491
 492 struct nvgre_encap_conf nvgre_encap_conf = {
 493         .select_ipv4 = 1,
 494         .select_vlan = 0,
 495         .tni = "\x00\x00\x00",
 496         .ipv4_src = IPv4(127, 0, 0, 1),
 497         .ipv4_dst = IPv4(255, 255, 255, 255),
 498         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 499                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 500         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 501                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 502         .vlan_tci = 0,
 503         .eth_src = "\x00\x00\x00\x00\x00\x00",
 504         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 505 };
 506
 507 /* Forward function declarations */
 508 static void setup_attached_port(portid_t pi);
 509 static void map_port_queue_stats_mapping_registers(portid_t pi,
 510                                                    struct rte_port *port);
 511 static void check_all_ports_link_status(uint32_t port_mask);
 512 static int eth_event_callback(portid_t port_id,
 513                               enum rte_eth_event_type type,
 514                               void *param, void *ret_param);
 515 static void dev_event_callback(const char *device_name,
 516                                 enum rte_dev_event_type type,
 517                                 void *param);
 518
 519 /*
 520  * Check if all the ports are started.
 521  * If yes, return positive value. If not, return zero.
 522  */
 523 static int all_ports_started(void);
 524
 525 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 526 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN;
 527
 528 /*
 529  * Helper function to check if socket is already discovered.
 530  * If yes, return positive value. If not, return zero.
 531  */
 532 int
 533 new_socket_id(unsigned int socket_id)
 534 {
 535         unsigned int i;
 536
 537         for (i = 0; i < num_sockets; i++) {
 538                 if (socket_ids[i] == socket_id)
 539                         return 0;
 540         }
 541         return 1;
 542 }
 543
 544 /*
 545  * Setup default configuration.
 546  */
 547 static void
 548 set_default_fwd_lcores_config(void)
 549 {
 550         unsigned int i;
 551         unsigned int nb_lc;
 552         unsigned int sock_num;
 553
 554         nb_lc = 0;
 555         for (i = 0; i < RTE_MAX_LCORE; i++) {
 556                 if (!rte_lcore_is_enabled(i))
 557                         continue;
 558                 sock_num = rte_lcore_to_socket_id(i);
 559                 if (new_socket_id(sock_num)) {
 560                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 561                                 rte_exit(EXIT_FAILURE,
 562                                          "Total sockets greater than %u\n",
 563                                          RTE_MAX_NUMA_NODES);
 564                         }
 565                         socket_ids[num_sockets++] = sock_num;
 566                 }
 567                 if (i == rte_get_master_lcore())
 568                         continue;
 569                 fwd_lcores_cpuids[nb_lc++] = i;
 570         }
 571         nb_lcores = (lcoreid_t) nb_lc;
 572         nb_cfg_lcores = nb_lcores;
 573         nb_fwd_lcores = 1;
 574 }
 575
 576 static void
 577 set_def_peer_eth_addrs(void)
 578 {
 579         portid_t i;
 580
 581         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 582                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 583                 peer_eth_addrs[i].addr_bytes[5] = i;
 584         }
 585 }
 586
 587 static void
 588 set_default_fwd_ports_config(void)
 589 {
 590         portid_t pt_id;
 591         int i = 0;
 592
 593         RTE_ETH_FOREACH_DEV(pt_id) {
 594                 fwd_ports_ids[i++] = pt_id;
 595
 596                 /* Update sockets info according to the attached device */
 597                 int socket_id = rte_eth_dev_socket_id(pt_id);
 598                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 599                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 600                                 rte_exit(EXIT_FAILURE,
 601                                          "Total sockets greater than %u\n",
 602                                          RTE_MAX_NUMA_NODES);
 603                         }
 604                         socket_ids[num_sockets++] = socket_id;
 605                 }
 606         }
 607
 608         nb_cfg_ports = nb_ports;
 609         nb_fwd_ports = nb_ports;
 610 }
 611
 612 void
 613 set_def_fwd_config(void)
 614 {
 615         set_default_fwd_lcores_config();
 616         set_def_peer_eth_addrs();
 617         set_default_fwd_ports_config();
 618 }
 619
 620 /* extremely pessimistic estimation of memory required to create a mempool */
 621 static int
 622 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 623 {
 624         unsigned int n_pages, mbuf_per_pg, leftover;
 625         uint64_t total_mem, mbuf_mem, obj_sz;
 626
 627         /* there is no good way to predict how much space the mempool will
 628          * occupy because it will allocate chunks on the fly, and some of those
 629          * will come from default DPDK memory while some will come from our
 630          * external memory, so just assume 128MB will be enough for everyone.
 631          */
 632         uint64_t hdr_mem = 128 << 20;
 633
 634         /* account for possible non-contiguousness */
 635         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 636         if (obj_sz > pgsz) {
 637                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 638                 return -1;
 639         }
 640
 641         mbuf_per_pg = pgsz / obj_sz;
 642         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 643         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 644
 645         mbuf_mem = n_pages * pgsz;
 646
 647         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 648
 649         if (total_mem > SIZE_MAX) {
 650                 TESTPMD_LOG(ERR, "Memory size too big\n");
 651                 return -1;
 652         }
 653         *out = (size_t)total_mem;
 654
 655         return 0;
 656 }
 657
 658 static int
 659 pagesz_flags(uint64_t page_sz)
 660 {
 661         /* as per mmap() manpage, all page sizes are log2 of page size
 662          * shifted by MAP_HUGE_SHIFT
 663          */
 664         int log2 = rte_log2_u64(page_sz);
 665
 666         return (log2 << HUGE_SHIFT);
 667 }
 668
 669 static void *
 670 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 671 {
 672         void *addr;
 673         int flags;
 674
 675         /* allocate anonymous hugepages */
 676         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 677         if (huge)
 678                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 679
 680         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 681         if (addr == MAP_FAILED)
 682                 return NULL;
 683
 684         return addr;
 685 }
 686
 687 struct extmem_param {
 688         void *addr;
 689         size_t len;
 690         size_t pgsz;
 691         rte_iova_t *iova_table;
 692         unsigned int iova_table_len;
 693 };
 694
 695 static int
 696 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 697                 bool huge)
 698 {
 699         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 700                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 701         unsigned int cur_page, n_pages, pgsz_idx;
 702         size_t mem_sz, cur_pgsz;
 703         rte_iova_t *iovas = NULL;
 704         void *addr;
 705         int ret;
 706
 707         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 708                 /* skip anything that is too big */
 709                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 710                         continue;
 711
 712                 cur_pgsz = pgsizes[pgsz_idx];
 713
 714                 /* if we were told not to allocate hugepages, override */
 715                 if (!huge)
 716                         cur_pgsz = sysconf(_SC_PAGESIZE);
 717
 718                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 719                 if (ret < 0) {
 720                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 721                         return -1;
 722                 }
 723
 724                 /* allocate our memory */
 725                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 726
 727                 /* if we couldn't allocate memory with a specified page size,
 728                  * that doesn't mean we can't do it with other page sizes, so
 729                  * try another one.
 730                  */
 731                 if (addr == NULL)
 732                         continue;
 733
 734                 /* store IOVA addresses for every page in this memory area */
 735                 n_pages = mem_sz / cur_pgsz;
 736
 737                 iovas = malloc(sizeof(*iovas) * n_pages);
 738
 739                 if (iovas == NULL) {
 740                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 741                         goto fail;
 742                 }
 743                 /* lock memory if it's not huge pages */
 744                 if (!huge)
 745                         mlock(addr, mem_sz);
 746
 747                 /* populate IOVA addresses */
 748                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 749                         rte_iova_t iova;
 750                         size_t offset;
 751                         void *cur;
 752
 753                         offset = cur_pgsz * cur_page;
 754                         cur = RTE_PTR_ADD(addr, offset);
 755
 756                         /* touch the page before getting its IOVA */
 757                         *(volatile char *)cur = 0;
 758
 759                         iova = rte_mem_virt2iova(cur);
 760
 761                         iovas[cur_page] = iova;
 762                 }
 763
 764                 break;
 765         }
 766         /* if we couldn't allocate anything */
 767         if (iovas == NULL)
 768                 return -1;
 769
 770         param->addr = addr;
 771         param->len = mem_sz;
 772         param->pgsz = cur_pgsz;
 773         param->iova_table = iovas;
 774         param->iova_table_len = n_pages;
 775
 776         return 0;
 777 fail:
 778         if (iovas)
 779                 free(iovas);
 780         if (addr)
 781                 munmap(addr, mem_sz);
 782
 783         return -1;
 784 }
 785
 786 static int
 787 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 788 {
 789         struct extmem_param param;
 790         int socket_id, ret;
 791
 792         memset(&param, 0, sizeof(param));
 793
 794         /* check if our heap exists */
 795         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 796         if (socket_id < 0) {
 797                 /* create our heap */
 798                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 799                 if (ret < 0) {
 800                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 801                         return -1;
 802                 }
 803         }
 804
 805         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 806         if (ret < 0) {
 807                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 808                 return -1;
 809         }
 810
 811         /* we now have a valid memory area, so add it to heap */
 812         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 813                         param.addr, param.len, param.iova_table,
 814                         param.iova_table_len, param.pgsz);
 815
 816         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 817
 818         /* not needed any more */
 819         free(param.iova_table);
 820
 821         if (ret < 0) {
 822                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 823                 munmap(param.addr, param.len);
 824                 return -1;
 825         }
 826
 827         /* success */
 828
 829         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 830                         param.len >> 20);
 831
 832         return 0;
 833 }
 834
 835 /*
 836  * Configuration initialisation done once at init time.
 837  */
 838 static void
 839 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 840                  unsigned int socket_id)
 841 {
 842         char pool_name[RTE_MEMPOOL_NAMESIZE];
 843         struct rte_mempool *rte_mp = NULL;
 844         uint32_t mb_size;
 845
 846         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 847         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 848
 849         TESTPMD_LOG(INFO,
 850                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 851                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 852
 853         switch (mp_alloc_type) {
 854         case MP_ALLOC_NATIVE:
 855                 {
 856                         /* wrapper to rte_mempool_create() */
 857                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 858                                         rte_mbuf_best_mempool_ops());
 859                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 860                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 861                         break;
 862                 }
 863         case MP_ALLOC_ANON:
 864                 {
 865                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 866                                 mb_size, (unsigned int) mb_mempool_cache,
 867                                 sizeof(struct rte_pktmbuf_pool_private),
 868                                 socket_id, 0);
 869                         if (rte_mp == NULL)
 870                                 goto err;
 871
 872                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 873                                 rte_mempool_free(rte_mp);
 874                                 rte_mp = NULL;
 875                                 goto err;
 876                         }
 877                         rte_pktmbuf_pool_init(rte_mp, NULL);
 878                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 879                         break;
 880                 }
 881         case MP_ALLOC_XMEM:
 882         case MP_ALLOC_XMEM_HUGE:
 883                 {
 884                         int heap_socket;
 885                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 886
 887                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 888                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 889
 890                         heap_socket =
 891                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 892                         if (heap_socket < 0)
 893                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 894
 895                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 896                                         rte_mbuf_best_mempool_ops());
 897                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 898                                         mb_mempool_cache, 0, mbuf_seg_size,
 899                                         heap_socket);
 900                         break;
 901                 }
 902         default:
 903                 {
 904                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 905                 }
 906         }
 907
 908 err:
 909         if (rte_mp == NULL) {
 910                 rte_exit(EXIT_FAILURE,
 911                         "Creation of mbuf pool for socket %u failed: %s\n",
 912                         socket_id, rte_strerror(rte_errno));
 913         } else if (verbose_level > 0) {
 914                 rte_mempool_dump(stdout, rte_mp);
 915         }
 916 }
 917
 918 /*
 919  * Check given socket id is valid or not with NUMA mode,
 920  * if valid, return 0, else return -1
 921  */
 922 static int
 923 check_socket_id(const unsigned int socket_id)
 924 {
 925         static int warning_once = 0;
 926
 927         if (new_socket_id(socket_id)) {
 928                 if (!warning_once && numa_support)
 929                         printf("Warning: NUMA should be configured manually by"
 930                                " using --port-numa-config and"
 931                                " --ring-numa-config parameters along with"
 932                                " --numa.\n");
 933                 warning_once = 1;
 934                 return -1;
 935         }
 936         return 0;
 937 }
 938
 939 /*
 940  * Get the allowed maximum number of RX queues.
 941  * *pid return the port id which has minimal value of
 942  * max_rx_queues in all ports.
 943  */
 944 queueid_t
 945 get_allowed_max_nb_rxq(portid_t *pid)
 946 {
 947         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
 948         portid_t pi;
 949         struct rte_eth_dev_info dev_info;
 950
 951         RTE_ETH_FOREACH_DEV(pi) {
 952                 rte_eth_dev_info_get(pi, &dev_info);
 953                 if (dev_info.max_rx_queues < allowed_max_rxq) {
 954                         allowed_max_rxq = dev_info.max_rx_queues;
 955                         *pid = pi;
 956                 }
 957         }
 958         return allowed_max_rxq;
 959 }
 960
 961 /*
 962  * Check input rxq is valid or not.
 963  * If input rxq is not greater than any of maximum number
 964  * of RX queues of all ports, it is valid.
 965  * if valid, return 0, else return -1
 966  */
 967 int
 968 check_nb_rxq(queueid_t rxq)
 969 {
 970         queueid_t allowed_max_rxq;
 971         portid_t pid = 0;
 972
 973         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
 974         if (rxq > allowed_max_rxq) {
 975                 printf("Fail: input rxq (%u) can't be greater "
 976                        "than max_rx_queues (%u) of port %u\n",
 977                        rxq,
 978                        allowed_max_rxq,
 979                        pid);
 980                 return -1;
 981         }
 982         return 0;
 983 }
 984
 985 /*
 986  * Get the allowed maximum number of TX queues.
 987  * *pid return the port id which has minimal value of
 988  * max_tx_queues in all ports.
 989  */
 990 queueid_t
 991 get_allowed_max_nb_txq(portid_t *pid)
 992 {
 993         queueid_t allowed_max_txq = MAX_QUEUE_ID;
 994         portid_t pi;
 995         struct rte_eth_dev_info dev_info;
 996
 997         RTE_ETH_FOREACH_DEV(pi) {
 998                 rte_eth_dev_info_get(pi, &dev_info);
 999                 if (dev_info.max_tx_queues < allowed_max_txq) {
1000                         allowed_max_txq = dev_info.max_tx_queues;
1001                         *pid = pi;
1002                 }
1003         }
1004         return allowed_max_txq;
1005 }
1006
1007 /*
1008  * Check input txq is valid or not.
1009  * If input txq is not greater than any of maximum number
1010  * of TX queues of all ports, it is valid.
1011  * if valid, return 0, else return -1
1012  */
1013 int
1014 check_nb_txq(queueid_t txq)
1015 {
1016         queueid_t allowed_max_txq;
1017         portid_t pid = 0;
1018
1019         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1020         if (txq > allowed_max_txq) {
1021                 printf("Fail: input txq (%u) can't be greater "
1022                        "than max_tx_queues (%u) of port %u\n",
1023                        txq,
1024                        allowed_max_txq,
1025                        pid);
1026                 return -1;
1027         }
1028         return 0;
1029 }
1030
1031 static void
1032 init_config(void)
1033 {
1034         portid_t pid;
1035         struct rte_port *port;
1036         struct rte_mempool *mbp;
1037         unsigned int nb_mbuf_per_pool;
1038         lcoreid_t  lc_id;
1039         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1040         struct rte_gro_param gro_param;
1041         uint32_t gso_types;
1042         int k;
1043
1044         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1045
1046         /* Configuration of logical cores. */
1047         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1048                                 sizeof(struct fwd_lcore *) * nb_lcores,
1049                                 RTE_CACHE_LINE_SIZE);
1050         if (fwd_lcores == NULL) {
1051                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1052                                                         "failed\n", nb_lcores);
1053         }
1054         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1055                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1056                                                sizeof(struct fwd_lcore),
1057                                                RTE_CACHE_LINE_SIZE);
1058                 if (fwd_lcores[lc_id] == NULL) {
1059                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1060                                                                 "failed\n");
1061                 }
1062                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1063         }
1064
1065         RTE_ETH_FOREACH_DEV(pid) {
1066                 port = &ports[pid];
1067                 /* Apply default TxRx configuration for all ports */
1068                 port->dev_conf.txmode = tx_mode;
1069                 port->dev_conf.rxmode = rx_mode;
1070                 rte_eth_dev_info_get(pid, &port->dev_info);
1071
1072                 if (!(port->dev_info.tx_offload_capa &
1073                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1074                         port->dev_conf.txmode.offloads &=
1075                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1076                 if (!(port->dev_info.tx_offload_capa &
1077                         DEV_TX_OFFLOAD_MATCH_METADATA))
1078                         port->dev_conf.txmode.offloads &=
1079                                 ~DEV_TX_OFFLOAD_MATCH_METADATA;
1080                 if (numa_support) {
1081                         if (port_numa[pid] != NUMA_NO_CONFIG)
1082                                 port_per_socket[port_numa[pid]]++;
1083                         else {
1084                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1085
1086                                 /*
1087                                  * if socket_id is invalid,
1088                                  * set to the first available socket.
1089                                  */
1090                                 if (check_socket_id(socket_id) < 0)
1091                                         socket_id = socket_ids[0];
1092                                 port_per_socket[socket_id]++;
1093                         }
1094                 }
1095
1096                 /* Apply Rx offloads configuration */
1097                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1098                         port->rx_conf[k].offloads =
1099                                 port->dev_conf.rxmode.offloads;
1100                 /* Apply Tx offloads configuration */
1101                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1102                         port->tx_conf[k].offloads =
1103                                 port->dev_conf.txmode.offloads;
1104
1105                 /* set flag to initialize port/queue */
1106                 port->need_reconfig = 1;
1107                 port->need_reconfig_queues = 1;
1108                 port->tx_metadata = 0;
1109         }
1110
1111         /*
1112          * Create pools of mbuf.
1113          * If NUMA support is disabled, create a single pool of mbuf in
1114          * socket 0 memory by default.
1115          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1116          *
1117          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1118          * nb_txd can be configured at run time.
1119          */
1120         if (param_total_num_mbufs)
1121                 nb_mbuf_per_pool = param_total_num_mbufs;
1122         else {
1123                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1124                         (nb_lcores * mb_mempool_cache) +
1125                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1126                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1127         }
1128
1129         if (numa_support) {
1130                 uint8_t i;
1131
1132                 for (i = 0; i < num_sockets; i++)
1133                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
1134                                          socket_ids[i]);
1135         } else {
1136                 if (socket_num == UMA_NO_CONFIG)
1137                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0);
1138                 else
1139                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
1140                                                  socket_num);
1141         }
1142
1143         init_port_config();
1144
1145         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1146                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1147         /*
1148          * Records which Mbuf pool to use by each logical core, if needed.
1149          */
1150         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1151                 mbp = mbuf_pool_find(
1152                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1153
1154                 if (mbp == NULL)
1155                         mbp = mbuf_pool_find(0);
1156                 fwd_lcores[lc_id]->mbp = mbp;
1157                 /* initialize GSO context */
1158                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1159                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1160                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1161                 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN -
1162                         ETHER_CRC_LEN;
1163                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1164         }
1165
1166         /* Configuration of packet forwarding streams. */
1167         if (init_fwd_streams() < 0)
1168                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1169
1170         fwd_config_setup();
1171
1172         /* create a gro context for each lcore */
1173         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1174         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1175         gro_param.max_item_per_flow = MAX_PKT_BURST;
1176         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1177                 gro_param.socket_id = rte_lcore_to_socket_id(
1178                                 fwd_lcores_cpuids[lc_id]);
1179                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1180                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1181                         rte_exit(EXIT_FAILURE,
1182                                         "rte_gro_ctx_create() failed\n");
1183                 }
1184         }
1185
1186 #if defined RTE_LIBRTE_PMD_SOFTNIC
1187         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1188                 RTE_ETH_FOREACH_DEV(pid) {
1189                         port = &ports[pid];
1190                         const char *driver = port->dev_info.driver_name;
1191
1192                         if (strcmp(driver, "net_softnic") == 0)
1193                                 port->softport.fwd_lcore_arg = fwd_lcores;
1194                 }
1195         }
1196 #endif
1197
1198 }
1199
1200
1201 void
1202 reconfig(portid_t new_port_id, unsigned socket_id)
1203 {
1204         struct rte_port *port;
1205
1206         /* Reconfiguration of Ethernet ports. */
1207         port = &ports[new_port_id];
1208         rte_eth_dev_info_get(new_port_id, &port->dev_info);
1209
1210         /* set flag to initialize port/queue */
1211         port->need_reconfig = 1;
1212         port->need_reconfig_queues = 1;
1213         port->socket_id = socket_id;
1214
1215         init_port_config();
1216 }
1217
1218
1219 int
1220 init_fwd_streams(void)
1221 {
1222         portid_t pid;
1223         struct rte_port *port;
1224         streamid_t sm_id, nb_fwd_streams_new;
1225         queueid_t q;
1226
1227         /* set socket id according to numa or not */
1228         RTE_ETH_FOREACH_DEV(pid) {
1229                 port = &ports[pid];
1230                 if (nb_rxq > port->dev_info.max_rx_queues) {
1231                         printf("Fail: nb_rxq(%d) is greater than "
1232                                 "max_rx_queues(%d)\n", nb_rxq,
1233                                 port->dev_info.max_rx_queues);
1234                         return -1;
1235                 }
1236                 if (nb_txq > port->dev_info.max_tx_queues) {
1237                         printf("Fail: nb_txq(%d) is greater than "
1238                                 "max_tx_queues(%d)\n", nb_txq,
1239                                 port->dev_info.max_tx_queues);
1240                         return -1;
1241                 }
1242                 if (numa_support) {
1243                         if (port_numa[pid] != NUMA_NO_CONFIG)
1244                                 port->socket_id = port_numa[pid];
1245                         else {
1246                                 port->socket_id = rte_eth_dev_socket_id(pid);
1247
1248                                 /*
1249                                  * if socket_id is invalid,
1250                                  * set to the first available socket.
1251                                  */
1252                                 if (check_socket_id(port->socket_id) < 0)
1253                                         port->socket_id = socket_ids[0];
1254                         }
1255                 }
1256                 else {
1257                         if (socket_num == UMA_NO_CONFIG)
1258                                 port->socket_id = 0;
1259                         else
1260                                 port->socket_id = socket_num;
1261                 }
1262         }
1263
1264         q = RTE_MAX(nb_rxq, nb_txq);
1265         if (q == 0) {
1266                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1267                 return -1;
1268         }
1269         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1270         if (nb_fwd_streams_new == nb_fwd_streams)
1271                 return 0;
1272         /* clear the old */
1273         if (fwd_streams != NULL) {
1274                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1275                         if (fwd_streams[sm_id] == NULL)
1276                                 continue;
1277                         rte_free(fwd_streams[sm_id]);
1278                         fwd_streams[sm_id] = NULL;
1279                 }
1280                 rte_free(fwd_streams);
1281                 fwd_streams = NULL;
1282         }
1283
1284         /* init new */
1285         nb_fwd_streams = nb_fwd_streams_new;
1286         if (nb_fwd_streams) {
1287                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1288                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1289                         RTE_CACHE_LINE_SIZE);
1290                 if (fwd_streams == NULL)
1291                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1292                                  " (struct fwd_stream *)) failed\n",
1293                                  nb_fwd_streams);
1294
1295                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1296                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1297                                 " struct fwd_stream", sizeof(struct fwd_stream),
1298                                 RTE_CACHE_LINE_SIZE);
1299                         if (fwd_streams[sm_id] == NULL)
1300                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1301                                          "(struct fwd_stream) failed\n");
1302                 }
1303         }
1304
1305         return 0;
1306 }
1307
1308 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1309 static void
1310 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1311 {
1312         unsigned int total_burst;
1313         unsigned int nb_burst;
1314         unsigned int burst_stats[3];
1315         uint16_t pktnb_stats[3];
1316         uint16_t nb_pkt;
1317         int burst_percent[3];
1318
1319         /*
1320          * First compute the total number of packet bursts and the
1321          * two highest numbers of bursts of the same number of packets.
1322          */
1323         total_burst = 0;
1324         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1325         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1326         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1327                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1328                 if (nb_burst == 0)
1329                         continue;
1330                 total_burst += nb_burst;
1331                 if (nb_burst > burst_stats[0]) {
1332                         burst_stats[1] = burst_stats[0];
1333                         pktnb_stats[1] = pktnb_stats[0];
1334                         burst_stats[0] = nb_burst;
1335                         pktnb_stats[0] = nb_pkt;
1336                 } else if (nb_burst > burst_stats[1]) {
1337                         burst_stats[1] = nb_burst;
1338                         pktnb_stats[1] = nb_pkt;
1339                 }
1340         }
1341         if (total_burst == 0)
1342                 return;
1343         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1344         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1345                burst_percent[0], (int) pktnb_stats[0]);
1346         if (burst_stats[0] == total_burst) {
1347                 printf("]\n");
1348                 return;
1349         }
1350         if (burst_stats[0] + burst_stats[1] == total_burst) {
1351                 printf(" + %d%% of %d pkts]\n",
1352                        100 - burst_percent[0], pktnb_stats[1]);
1353                 return;
1354         }
1355         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1356         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1357         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1358                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1359                 return;
1360         }
1361         printf(" + %d%% of %d pkts + %d%% of others]\n",
1362                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1363 }
1364 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1365
1366 static void
1367 fwd_stream_stats_display(streamid_t stream_id)
1368 {
1369         struct fwd_stream *fs;
1370         static const char *fwd_top_stats_border = "-------";
1371
1372         fs = fwd_streams[stream_id];
1373         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1374             (fs->fwd_dropped == 0))
1375                 return;
1376         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1377                "TX Port=%2d/Queue=%2d %s\n",
1378                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1379                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1380         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1381                " TX-dropped: %-14"PRIu64,
1382                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1383
1384         /* if checksum mode */
1385         if (cur_fwd_eng == &csum_fwd_engine) {
1386                 printf("  RX- bad IP checksum: %-14"PRIu64
1387                        "  Rx- bad L4 checksum: %-14"PRIu64
1388                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1389                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1390                         fs->rx_bad_outer_l4_csum);
1391         } else {
1392                 printf("\n");
1393         }
1394
1395 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1396         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1397         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1398 #endif
1399 }
1400
1401 void
1402 fwd_stats_display(void)
1403 {
1404         static const char *fwd_stats_border = "----------------------";
1405         static const char *acc_stats_border = "+++++++++++++++";
1406         struct {
1407                 struct fwd_stream *rx_stream;
1408                 struct fwd_stream *tx_stream;
1409                 uint64_t tx_dropped;
1410                 uint64_t rx_bad_ip_csum;
1411                 uint64_t rx_bad_l4_csum;
1412                 uint64_t rx_bad_outer_l4_csum;
1413         } ports_stats[RTE_MAX_ETHPORTS];
1414         uint64_t total_rx_dropped = 0;
1415         uint64_t total_tx_dropped = 0;
1416         uint64_t total_rx_nombuf = 0;
1417         struct rte_eth_stats stats;
1418 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1419         uint64_t fwd_cycles = 0;
1420 #endif
1421         uint64_t total_recv = 0;
1422         uint64_t total_xmit = 0;
1423         struct rte_port *port;
1424         streamid_t sm_id;
1425         portid_t pt_id;
1426         int i;
1427
1428         memset(ports_stats, 0, sizeof(ports_stats));
1429
1430         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1431                 struct fwd_stream *fs = fwd_streams[sm_id];
1432
1433                 if (cur_fwd_config.nb_fwd_streams >
1434                     cur_fwd_config.nb_fwd_ports) {
1435                         fwd_stream_stats_display(sm_id);
1436                 } else {
1437                         ports_stats[fs->tx_port].tx_stream = fs;
1438                         ports_stats[fs->rx_port].rx_stream = fs;
1439                 }
1440
1441                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1442
1443                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1444                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1445                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1446                                 fs->rx_bad_outer_l4_csum;
1447
1448 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1449                 fwd_cycles += fs->core_cycles;
1450 #endif
1451         }
1452         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1453                 uint8_t j;
1454
1455                 pt_id = fwd_ports_ids[i];
1456                 port = &ports[pt_id];
1457
1458                 rte_eth_stats_get(pt_id, &stats);
1459                 stats.ipackets -= port->stats.ipackets;
1460                 stats.opackets -= port->stats.opackets;
1461                 stats.ibytes -= port->stats.ibytes;
1462                 stats.obytes -= port->stats.obytes;
1463                 stats.imissed -= port->stats.imissed;
1464                 stats.oerrors -= port->stats.oerrors;
1465                 stats.rx_nombuf -= port->stats.rx_nombuf;
1466
1467                 total_recv += stats.ipackets;
1468                 total_xmit += stats.opackets;
1469                 total_rx_dropped += stats.imissed;
1470                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1471                 total_tx_dropped += stats.oerrors;
1472                 total_rx_nombuf  += stats.rx_nombuf;
1473
1474                 printf("\n  %s Forward statistics for port %-2d %s\n",
1475                        fwd_stats_border, pt_id, fwd_stats_border);
1476
1477                 if (!port->rx_queue_stats_mapping_enabled &&
1478                     !port->tx_queue_stats_mapping_enabled) {
1479                         printf("  RX-packets: %-14"PRIu64
1480                                " RX-dropped: %-14"PRIu64
1481                                "RX-total: %-"PRIu64"\n",
1482                                stats.ipackets, stats.imissed,
1483                                stats.ipackets + stats.imissed);
1484
1485                         if (cur_fwd_eng == &csum_fwd_engine)
1486                                 printf("  Bad-ipcsum: %-14"PRIu64
1487                                        " Bad-l4csum: %-14"PRIu64
1488                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1489                                        ports_stats[pt_id].rx_bad_ip_csum,
1490                                        ports_stats[pt_id].rx_bad_l4_csum,
1491                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1492                         if (stats.ierrors + stats.rx_nombuf > 0) {
1493                                 printf("  RX-error: %-"PRIu64"\n",
1494                                        stats.ierrors);
1495                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1496                                        stats.rx_nombuf);
1497                         }
1498
1499                         printf("  TX-packets: %-14"PRIu64
1500                                " TX-dropped: %-14"PRIu64
1501                                "TX-total: %-"PRIu64"\n",
1502                                stats.opackets, ports_stats[pt_id].tx_dropped,
1503                                stats.opackets + ports_stats[pt_id].tx_dropped);
1504                 } else {
1505                         printf("  RX-packets:             %14"PRIu64
1506                                "    RX-dropped:%14"PRIu64
1507                                "    RX-total:%14"PRIu64"\n",
1508                                stats.ipackets, stats.imissed,
1509                                stats.ipackets + stats.imissed);
1510
1511                         if (cur_fwd_eng == &csum_fwd_engine)
1512                                 printf("  Bad-ipcsum:%14"PRIu64
1513                                        "    Bad-l4csum:%14"PRIu64
1514                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1515                                        ports_stats[pt_id].rx_bad_ip_csum,
1516                                        ports_stats[pt_id].rx_bad_l4_csum,
1517                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1518                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1519                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1520                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1521                                        stats.rx_nombuf);
1522                         }
1523
1524                         printf("  TX-packets:             %14"PRIu64
1525                                "    TX-dropped:%14"PRIu64
1526                                "    TX-total:%14"PRIu64"\n",
1527                                stats.opackets, ports_stats[pt_id].tx_dropped,
1528                                stats.opackets + ports_stats[pt_id].tx_dropped);
1529                 }
1530
1531 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1532                 if (ports_stats[pt_id].rx_stream)
1533                         pkt_burst_stats_display("RX",
1534                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1535                 if (ports_stats[pt_id].tx_stream)
1536                         pkt_burst_stats_display("TX",
1537                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1538 #endif
1539
1540                 if (port->rx_queue_stats_mapping_enabled) {
1541                         printf("\n");
1542                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1543                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1544                                        "     RX-errors:%14"PRIu64
1545                                        "    RX-bytes:%14"PRIu64"\n",
1546                                        j, stats.q_ipackets[j],
1547                                        stats.q_errors[j], stats.q_ibytes[j]);
1548                         }
1549                         printf("\n");
1550                 }
1551                 if (port->tx_queue_stats_mapping_enabled) {
1552                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1553                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1554                                        "                                 TX-bytes:%14"
1555                                        PRIu64"\n",
1556                                        j, stats.q_opackets[j],
1557                                        stats.q_obytes[j]);
1558                         }
1559                 }
1560
1561                 printf("  %s--------------------------------%s\n",
1562                        fwd_stats_border, fwd_stats_border);
1563         }
1564
1565         printf("\n  %s Accumulated forward statistics for all ports"
1566                "%s\n",
1567                acc_stats_border, acc_stats_border);
1568         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1569                "%-"PRIu64"\n"
1570                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1571                "%-"PRIu64"\n",
1572                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1573                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1574         if (total_rx_nombuf > 0)
1575                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1576         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1577                "%s\n",
1578                acc_stats_border, acc_stats_border);
1579 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1580         if (total_recv > 0)
1581                 printf("\n  CPU cycles/packet=%u (total cycles="
1582                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1583                        (unsigned int)(fwd_cycles / total_recv),
1584                        fwd_cycles, total_recv);
1585 #endif
1586 }
1587
1588 void
1589 fwd_stats_reset(void)
1590 {
1591         streamid_t sm_id;
1592         portid_t pt_id;
1593         int i;
1594
1595         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1596                 pt_id = fwd_ports_ids[i];
1597                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1598         }
1599         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1600                 struct fwd_stream *fs = fwd_streams[sm_id];
1601
1602                 fs->rx_packets = 0;
1603                 fs->tx_packets = 0;
1604                 fs->fwd_dropped = 0;
1605                 fs->rx_bad_ip_csum = 0;
1606                 fs->rx_bad_l4_csum = 0;
1607                 fs->rx_bad_outer_l4_csum = 0;
1608
1609 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1610                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1611                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1612 #endif
1613 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1614                 fs->core_cycles = 0;
1615 #endif
1616         }
1617 }
1618
1619 static void
1620 flush_fwd_rx_queues(void)
1621 {
1622         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1623         portid_t  rxp;
1624         portid_t port_id;
1625         queueid_t rxq;
1626         uint16_t  nb_rx;
1627         uint16_t  i;
1628         uint8_t   j;
1629         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1630         uint64_t timer_period;
1631
1632         /* convert to number of cycles */
1633         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1634
1635         for (j = 0; j < 2; j++) {
1636                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1637                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1638                                 port_id = fwd_ports_ids[rxp];
1639                                 /**
1640                                 * testpmd can stuck in the below do while loop
1641                                 * if rte_eth_rx_burst() always returns nonzero
1642                                 * packets. So timer is added to exit this loop
1643                                 * after 1sec timer expiry.
1644                                 */
1645                                 prev_tsc = rte_rdtsc();
1646                                 do {
1647                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1648                                                 pkts_burst, MAX_PKT_BURST);
1649                                         for (i = 0; i < nb_rx; i++)
1650                                                 rte_pktmbuf_free(pkts_burst[i]);
1651
1652                                         cur_tsc = rte_rdtsc();
1653                                         diff_tsc = cur_tsc - prev_tsc;
1654                                         timer_tsc += diff_tsc;
1655                                 } while ((nb_rx > 0) &&
1656                                         (timer_tsc < timer_period));
1657                                 timer_tsc = 0;
1658                         }
1659                 }
1660                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1661         }
1662 }
1663
1664 static void
1665 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1666 {
1667         struct fwd_stream **fsm;
1668         streamid_t nb_fs;
1669         streamid_t sm_id;
1670 #ifdef RTE_LIBRTE_BITRATE
1671         uint64_t tics_per_1sec;
1672         uint64_t tics_datum;
1673         uint64_t tics_current;
1674         uint16_t i, cnt_ports;
1675
1676         cnt_ports = nb_ports;
1677         tics_datum = rte_rdtsc();
1678         tics_per_1sec = rte_get_timer_hz();
1679 #endif
1680         fsm = &fwd_streams[fc->stream_idx];
1681         nb_fs = fc->stream_nb;
1682         do {
1683                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1684                         (*pkt_fwd)(fsm[sm_id]);
1685 #ifdef RTE_LIBRTE_BITRATE
1686                 if (bitrate_enabled != 0 &&
1687                                 bitrate_lcore_id == rte_lcore_id()) {
1688                         tics_current = rte_rdtsc();
1689                         if (tics_current - tics_datum >= tics_per_1sec) {
1690                                 /* Periodic bitrate calculation */
1691                                 for (i = 0; i < cnt_ports; i++)
1692                                         rte_stats_bitrate_calc(bitrate_data,
1693                                                 ports_ids[i]);
1694                                 tics_datum = tics_current;
1695                         }
1696                 }
1697 #endif
1698 #ifdef RTE_LIBRTE_LATENCY_STATS
1699                 if (latencystats_enabled != 0 &&
1700                                 latencystats_lcore_id == rte_lcore_id())
1701                         rte_latencystats_update();
1702 #endif
1703
1704         } while (! fc->stopped);
1705 }
1706
1707 static int
1708 start_pkt_forward_on_core(void *fwd_arg)
1709 {
1710         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1711                              cur_fwd_config.fwd_eng->packet_fwd);
1712         return 0;
1713 }
1714
1715 /*
1716  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1717  * Used to start communication flows in network loopback test configurations.
1718  */
1719 static int
1720 run_one_txonly_burst_on_core(void *fwd_arg)
1721 {
1722         struct fwd_lcore *fwd_lc;
1723         struct fwd_lcore tmp_lcore;
1724
1725         fwd_lc = (struct fwd_lcore *) fwd_arg;
1726         tmp_lcore = *fwd_lc;
1727         tmp_lcore.stopped = 1;
1728         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1729         return 0;
1730 }
1731
1732 /*
1733  * Launch packet forwarding:
1734  *     - Setup per-port forwarding context.
1735  *     - launch logical cores with their forwarding configuration.
1736  */
1737 static void
1738 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1739 {
1740         port_fwd_begin_t port_fwd_begin;
1741         unsigned int i;
1742         unsigned int lc_id;
1743         int diag;
1744
1745         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1746         if (port_fwd_begin != NULL) {
1747                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1748                         (*port_fwd_begin)(fwd_ports_ids[i]);
1749         }
1750         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1751                 lc_id = fwd_lcores_cpuids[i];
1752                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1753                         fwd_lcores[i]->stopped = 0;
1754                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1755                                                      fwd_lcores[i], lc_id);
1756                         if (diag != 0)
1757                                 printf("launch lcore %u failed - diag=%d\n",
1758                                        lc_id, diag);
1759                 }
1760         }
1761 }
1762
1763 /*
1764  * Launch packet forwarding configuration.
1765  */
1766 void
1767 start_packet_forwarding(int with_tx_first)
1768 {
1769         port_fwd_begin_t port_fwd_begin;
1770         port_fwd_end_t  port_fwd_end;
1771         struct rte_port *port;
1772         unsigned int i;
1773         portid_t   pt_id;
1774
1775         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1776                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1777
1778         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1779                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1780
1781         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1782                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1783                 (!nb_rxq || !nb_txq))
1784                 rte_exit(EXIT_FAILURE,
1785                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1786                         cur_fwd_eng->fwd_mode_name);
1787
1788         if (all_ports_started() == 0) {
1789                 printf("Not all ports were started\n");
1790                 return;
1791         }
1792         if (test_done == 0) {
1793                 printf("Packet forwarding already started\n");
1794                 return;
1795         }
1796
1797
1798         if(dcb_test) {
1799                 for (i = 0; i < nb_fwd_ports; i++) {
1800                         pt_id = fwd_ports_ids[i];
1801                         port = &ports[pt_id];
1802                         if (!port->dcb_flag) {
1803                                 printf("In DCB mode, all forwarding ports must "
1804                                        "be configured in this mode.\n");
1805                                 return;
1806                         }
1807                 }
1808                 if (nb_fwd_lcores == 1) {
1809                         printf("In DCB mode,the nb forwarding cores "
1810                                "should be larger than 1.\n");
1811                         return;
1812                 }
1813         }
1814         test_done = 0;
1815
1816         fwd_config_setup();
1817
1818         if(!no_flush_rx)
1819                 flush_fwd_rx_queues();
1820
1821         pkt_fwd_config_display(&cur_fwd_config);
1822         rxtx_config_display();
1823
1824         fwd_stats_reset();
1825         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1826                 pt_id = fwd_ports_ids[i];
1827                 port = &ports[pt_id];
1828                 map_port_queue_stats_mapping_registers(pt_id, port);
1829         }
1830         if (with_tx_first) {
1831                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1832                 if (port_fwd_begin != NULL) {
1833                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1834                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1835                 }
1836                 while (with_tx_first--) {
1837                         launch_packet_forwarding(
1838                                         run_one_txonly_burst_on_core);
1839                         rte_eal_mp_wait_lcore();
1840                 }
1841                 port_fwd_end = tx_only_engine.port_fwd_end;
1842                 if (port_fwd_end != NULL) {
1843                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1844                                 (*port_fwd_end)(fwd_ports_ids[i]);
1845                 }
1846         }
1847         launch_packet_forwarding(start_pkt_forward_on_core);
1848 }
1849
1850 void
1851 stop_packet_forwarding(void)
1852 {
1853         port_fwd_end_t port_fwd_end;
1854         lcoreid_t lc_id;
1855         portid_t pt_id;
1856         int i;
1857
1858         if (test_done) {
1859                 printf("Packet forwarding not started\n");
1860                 return;
1861         }
1862         printf("Telling cores to stop...");
1863         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1864                 fwd_lcores[lc_id]->stopped = 1;
1865         printf("\nWaiting for lcores to finish...\n");
1866         rte_eal_mp_wait_lcore();
1867         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1868         if (port_fwd_end != NULL) {
1869                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1870                         pt_id = fwd_ports_ids[i];
1871                         (*port_fwd_end)(pt_id);
1872                 }
1873         }
1874
1875         fwd_stats_display();
1876
1877         printf("\nDone.\n");
1878         test_done = 1;
1879 }
1880
1881 void
1882 dev_set_link_up(portid_t pid)
1883 {
1884         if (rte_eth_dev_set_link_up(pid) < 0)
1885                 printf("\nSet link up fail.\n");
1886 }
1887
1888 void
1889 dev_set_link_down(portid_t pid)
1890 {
1891         if (rte_eth_dev_set_link_down(pid) < 0)
1892                 printf("\nSet link down fail.\n");
1893 }
1894
1895 static int
1896 all_ports_started(void)
1897 {
1898         portid_t pi;
1899         struct rte_port *port;
1900
1901         RTE_ETH_FOREACH_DEV(pi) {
1902                 port = &ports[pi];
1903                 /* Check if there is a port which is not started */
1904                 if ((port->port_status != RTE_PORT_STARTED) &&
1905                         (port->slave_flag == 0))
1906                         return 0;
1907         }
1908
1909         /* No port is not started */
1910         return 1;
1911 }
1912
1913 int
1914 port_is_stopped(portid_t port_id)
1915 {
1916         struct rte_port *port = &ports[port_id];
1917
1918         if ((port->port_status != RTE_PORT_STOPPED) &&
1919             (port->slave_flag == 0))
1920                 return 0;
1921         return 1;
1922 }
1923
1924 int
1925 all_ports_stopped(void)
1926 {
1927         portid_t pi;
1928
1929         RTE_ETH_FOREACH_DEV(pi) {
1930                 if (!port_is_stopped(pi))
1931                         return 0;
1932         }
1933
1934         return 1;
1935 }
1936
1937 int
1938 port_is_started(portid_t port_id)
1939 {
1940         if (port_id_is_invalid(port_id, ENABLED_WARN))
1941                 return 0;
1942
1943         if (ports[port_id].port_status != RTE_PORT_STARTED)
1944                 return 0;
1945
1946         return 1;
1947 }
1948
1949 int
1950 start_port(portid_t pid)
1951 {
1952         int diag, need_check_link_status = -1;
1953         portid_t pi;
1954         queueid_t qi;
1955         struct rte_port *port;
1956         struct ether_addr mac_addr;
1957
1958         if (port_id_is_invalid(pid, ENABLED_WARN))
1959                 return 0;
1960
1961         if(dcb_config)
1962                 dcb_test = 1;
1963         RTE_ETH_FOREACH_DEV(pi) {
1964                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1965                         continue;
1966
1967                 need_check_link_status = 0;
1968                 port = &ports[pi];
1969                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1970                                                  RTE_PORT_HANDLING) == 0) {
1971                         printf("Port %d is now not stopped\n", pi);
1972                         continue;
1973                 }
1974
1975                 if (port->need_reconfig > 0) {
1976                         port->need_reconfig = 0;
1977
1978                         if (flow_isolate_all) {
1979                                 int ret = port_flow_isolate(pi, 1);
1980                                 if (ret) {
1981                                         printf("Failed to apply isolated"
1982                                                " mode on port %d\n", pi);
1983                                         return -1;
1984                                 }
1985                         }
1986                         configure_rxtx_dump_callbacks(0);
1987                         printf("Configuring Port %d (socket %u)\n", pi,
1988                                         port->socket_id);
1989                         /* configure port */
1990                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1991                                                 &(port->dev_conf));
1992                         if (diag != 0) {
1993                                 if (rte_atomic16_cmpset(&(port->port_status),
1994                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1995                                         printf("Port %d can not be set back "
1996                                                         "to stopped\n", pi);
1997                                 printf("Fail to configure port %d\n", pi);
1998                                 /* try to reconfigure port next time */
1999                                 port->need_reconfig = 1;
2000                                 return -1;
2001                         }
2002                 }
2003                 if (port->need_reconfig_queues > 0) {
2004                         port->need_reconfig_queues = 0;
2005                         /* setup tx queues */
2006                         for (qi = 0; qi < nb_txq; qi++) {
2007                                 if ((numa_support) &&
2008                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2009                                         diag = rte_eth_tx_queue_setup(pi, qi,
2010                                                 port->nb_tx_desc[qi],
2011                                                 txring_numa[pi],
2012                                                 &(port->tx_conf[qi]));
2013                                 else
2014                                         diag = rte_eth_tx_queue_setup(pi, qi,
2015                                                 port->nb_tx_desc[qi],
2016                                                 port->socket_id,
2017                                                 &(port->tx_conf[qi]));
2018
2019                                 if (diag == 0)
2020                                         continue;
2021
2022                                 /* Fail to setup tx queue, return */
2023                                 if (rte_atomic16_cmpset(&(port->port_status),
2024                                                         RTE_PORT_HANDLING,
2025                                                         RTE_PORT_STOPPED) == 0)
2026                                         printf("Port %d can not be set back "
2027                                                         "to stopped\n", pi);
2028                                 printf("Fail to configure port %d tx queues\n",
2029                                        pi);
2030                                 /* try to reconfigure queues next time */
2031                                 port->need_reconfig_queues = 1;
2032                                 return -1;
2033                         }
2034                         for (qi = 0; qi < nb_rxq; qi++) {
2035                                 /* setup rx queues */
2036                                 if ((numa_support) &&
2037                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2038                                         struct rte_mempool * mp =
2039                                                 mbuf_pool_find(rxring_numa[pi]);
2040                                         if (mp == NULL) {
2041                                                 printf("Failed to setup RX queue:"
2042                                                         "No mempool allocation"
2043                                                         " on the socket %d\n",
2044                                                         rxring_numa[pi]);
2045                                                 return -1;
2046                                         }
2047
2048                                         diag = rte_eth_rx_queue_setup(pi, qi,
2049                                              port->nb_rx_desc[qi],
2050                                              rxring_numa[pi],
2051                                              &(port->rx_conf[qi]),
2052                                              mp);
2053                                 } else {
2054                                         struct rte_mempool *mp =
2055                                                 mbuf_pool_find(port->socket_id);
2056                                         if (mp == NULL) {
2057                                                 printf("Failed to setup RX queue:"
2058                                                         "No mempool allocation"
2059                                                         " on the socket %d\n",
2060                                                         port->socket_id);
2061                                                 return -1;
2062                                         }
2063                                         diag = rte_eth_rx_queue_setup(pi, qi,
2064                                              port->nb_rx_desc[qi],
2065                                              port->socket_id,
2066                                              &(port->rx_conf[qi]),
2067                                              mp);
2068                                 }
2069                                 if (diag == 0)
2070                                         continue;
2071
2072                                 /* Fail to setup rx queue, return */
2073                                 if (rte_atomic16_cmpset(&(port->port_status),
2074                                                         RTE_PORT_HANDLING,
2075                                                         RTE_PORT_STOPPED) == 0)
2076                                         printf("Port %d can not be set back "
2077                                                         "to stopped\n", pi);
2078                                 printf("Fail to configure port %d rx queues\n",
2079                                        pi);
2080                                 /* try to reconfigure queues next time */
2081                                 port->need_reconfig_queues = 1;
2082                                 return -1;
2083                         }
2084                 }
2085                 configure_rxtx_dump_callbacks(verbose_level);
2086                 /* start port */
2087                 if (rte_eth_dev_start(pi) < 0) {
2088                         printf("Fail to start port %d\n", pi);
2089
2090                         /* Fail to setup rx queue, return */
2091                         if (rte_atomic16_cmpset(&(port->port_status),
2092                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2093                                 printf("Port %d can not be set back to "
2094                                                         "stopped\n", pi);
2095                         continue;
2096                 }
2097
2098                 if (rte_atomic16_cmpset(&(port->port_status),
2099                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2100                         printf("Port %d can not be set into started\n", pi);
2101
2102                 rte_eth_macaddr_get(pi, &mac_addr);
2103                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2104                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2105                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2106                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2107
2108                 /* at least one port started, need checking link status */
2109                 need_check_link_status = 1;
2110         }
2111
2112         if (need_check_link_status == 1 && !no_link_check)
2113                 check_all_ports_link_status(RTE_PORT_ALL);
2114         else if (need_check_link_status == 0)
2115                 printf("Please stop the ports first\n");
2116
2117         printf("Done\n");
2118         return 0;
2119 }
2120
2121 void
2122 stop_port(portid_t pid)
2123 {
2124         portid_t pi;
2125         struct rte_port *port;
2126         int need_check_link_status = 0;
2127
2128         if (dcb_test) {
2129                 dcb_test = 0;
2130                 dcb_config = 0;
2131         }
2132
2133         if (port_id_is_invalid(pid, ENABLED_WARN))
2134                 return;
2135
2136         printf("Stopping ports...\n");
2137
2138         RTE_ETH_FOREACH_DEV(pi) {
2139                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2140                         continue;
2141
2142                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2143                         printf("Please remove port %d from forwarding configuration.\n", pi);
2144                         continue;
2145                 }
2146
2147                 if (port_is_bonding_slave(pi)) {
2148                         printf("Please remove port %d from bonded device.\n", pi);
2149                         continue;
2150                 }
2151
2152                 port = &ports[pi];
2153                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2154                                                 RTE_PORT_HANDLING) == 0)
2155                         continue;
2156
2157                 rte_eth_dev_stop(pi);
2158
2159                 if (rte_atomic16_cmpset(&(port->port_status),
2160                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2161                         printf("Port %d can not be set into stopped\n", pi);
2162                 need_check_link_status = 1;
2163         }
2164         if (need_check_link_status && !no_link_check)
2165                 check_all_ports_link_status(RTE_PORT_ALL);
2166
2167         printf("Done\n");
2168 }
2169
2170 static void
2171 remove_invalid_ports_in(portid_t *array, portid_t *total)
2172 {
2173         portid_t i;
2174         portid_t new_total = 0;
2175
2176         for (i = 0; i < *total; i++)
2177                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2178                         array[new_total] = array[i];
2179                         new_total++;
2180                 }
2181         *total = new_total;
2182 }
2183
2184 static void
2185 remove_invalid_ports(void)
2186 {
2187         remove_invalid_ports_in(ports_ids, &nb_ports);
2188         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2189         nb_cfg_ports = nb_fwd_ports;
2190 }
2191
2192 void
2193 close_port(portid_t pid)
2194 {
2195         portid_t pi;
2196         struct rte_port *port;
2197
2198         if (port_id_is_invalid(pid, ENABLED_WARN))
2199                 return;
2200
2201         printf("Closing ports...\n");
2202
2203         RTE_ETH_FOREACH_DEV(pi) {
2204                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2205                         continue;
2206
2207                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2208                         printf("Please remove port %d from forwarding configuration.\n", pi);
2209                         continue;
2210                 }
2211
2212                 if (port_is_bonding_slave(pi)) {
2213                         printf("Please remove port %d from bonded device.\n", pi);
2214                         continue;
2215                 }
2216
2217                 port = &ports[pi];
2218                 if (rte_atomic16_cmpset(&(port->port_status),
2219                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2220                         printf("Port %d is already closed\n", pi);
2221                         continue;
2222                 }
2223
2224                 if (rte_atomic16_cmpset(&(port->port_status),
2225                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2226                         printf("Port %d is now not stopped\n", pi);
2227                         continue;
2228                 }
2229
2230                 if (port->flow_list)
2231                         port_flow_flush(pi);
2232                 rte_eth_dev_close(pi);
2233
2234                 remove_invalid_ports();
2235
2236                 if (rte_atomic16_cmpset(&(port->port_status),
2237                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2238                         printf("Port %d cannot be set to closed\n", pi);
2239         }
2240
2241         printf("Done\n");
2242 }
2243
2244 void
2245 reset_port(portid_t pid)
2246 {
2247         int diag;
2248         portid_t pi;
2249         struct rte_port *port;
2250
2251         if (port_id_is_invalid(pid, ENABLED_WARN))
2252                 return;
2253
2254         printf("Resetting ports...\n");
2255
2256         RTE_ETH_FOREACH_DEV(pi) {
2257                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2258                         continue;
2259
2260                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2261                         printf("Please remove port %d from forwarding "
2262                                "configuration.\n", pi);
2263                         continue;
2264                 }
2265
2266                 if (port_is_bonding_slave(pi)) {
2267                         printf("Please remove port %d from bonded device.\n",
2268                                pi);
2269                         continue;
2270                 }
2271
2272                 diag = rte_eth_dev_reset(pi);
2273                 if (diag == 0) {
2274                         port = &ports[pi];
2275                         port->need_reconfig = 1;
2276                         port->need_reconfig_queues = 1;
2277                 } else {
2278                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2279                 }
2280         }
2281
2282         printf("Done\n");
2283 }
2284
2285 void
2286 attach_port(char *identifier)
2287 {
2288         portid_t pi;
2289         struct rte_dev_iterator iterator;
2290
2291         printf("Attaching a new port...\n");
2292
2293         if (identifier == NULL) {
2294                 printf("Invalid parameters are specified\n");
2295                 return;
2296         }
2297
2298         if (rte_dev_probe(identifier) != 0) {
2299                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2300                 return;
2301         }
2302
2303         /* first attach mode: event */
2304         if (setup_on_probe_event) {
2305                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2306                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2307                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2308                                         ports[pi].need_setup != 0)
2309                                 setup_attached_port(pi);
2310                 return;
2311         }
2312
2313         /* second attach mode: iterator */
2314         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2315                 /* setup ports matching the devargs used for probing */
2316                 if (port_is_forwarding(pi))
2317                         continue; /* port was already attached before */
2318                 setup_attached_port(pi);
2319         }
2320 }
2321
2322 static void
2323 setup_attached_port(portid_t pi)
2324 {
2325         unsigned int socket_id;
2326
2327         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2328         /* if socket_id is invalid, set to the first available socket. */
2329         if (check_socket_id(socket_id) < 0)
2330                 socket_id = socket_ids[0];
2331         reconfig(pi, socket_id);
2332         rte_eth_promiscuous_enable(pi);
2333
2334         ports_ids[nb_ports++] = pi;
2335         fwd_ports_ids[nb_fwd_ports++] = pi;
2336         nb_cfg_ports = nb_fwd_ports;
2337         ports[pi].need_setup = 0;
2338         ports[pi].port_status = RTE_PORT_STOPPED;
2339
2340         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2341         printf("Done\n");
2342 }
2343
2344 void
2345 detach_port_device(portid_t port_id)
2346 {
2347         struct rte_device *dev;
2348         portid_t sibling;
2349
2350         printf("Removing a device...\n");
2351
2352         dev = rte_eth_devices[port_id].device;
2353         if (dev == NULL) {
2354                 printf("Device already removed\n");
2355                 return;
2356         }
2357
2358         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2359                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2360                         printf("Port not stopped\n");
2361                         return;
2362                 }
2363                 printf("Port was not closed\n");
2364                 if (ports[port_id].flow_list)
2365                         port_flow_flush(port_id);
2366         }
2367
2368         if (rte_dev_remove(dev) != 0) {
2369                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2370                 return;
2371         }
2372
2373         RTE_ETH_FOREACH_DEV_SIBLING(sibling, port_id) {
2374                 /* reset mapping between old ports and removed device */
2375                 rte_eth_devices[sibling].device = NULL;
2376                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2377                         /* sibling ports are forced to be closed */
2378                         ports[sibling].port_status = RTE_PORT_CLOSED;
2379                         printf("Port %u is closed\n", sibling);
2380                 }
2381         }
2382
2383         remove_invalid_ports();
2384
2385         printf("Device of port %u is detached\n", port_id);
2386         printf("Now total ports is %d\n", nb_ports);
2387         printf("Done\n");
2388         return;
2389 }
2390
2391 void
2392 pmd_test_exit(void)
2393 {
2394         struct rte_device *device;
2395         portid_t pt_id;
2396         int ret;
2397
2398         if (test_done == 0)
2399                 stop_packet_forwarding();
2400
2401         if (ports != NULL) {
2402                 no_link_check = 1;
2403                 RTE_ETH_FOREACH_DEV(pt_id) {
2404                         printf("\nStopping port %d...\n", pt_id);
2405                         fflush(stdout);
2406                         stop_port(pt_id);
2407                 }
2408                 RTE_ETH_FOREACH_DEV(pt_id) {
2409                         printf("\nShutting down port %d...\n", pt_id);
2410                         fflush(stdout);
2411                         close_port(pt_id);
2412
2413                         /*
2414                          * This is a workaround to fix a virtio-user issue that
2415                          * requires to call clean-up routine to remove existing
2416                          * socket.
2417                          * This workaround valid only for testpmd, needs a fix
2418                          * valid for all applications.
2419                          * TODO: Implement proper resource cleanup
2420                          */
2421                         device = rte_eth_devices[pt_id].device;
2422                         if (device && !strcmp(device->driver->name, "net_virtio_user"))
2423                                 detach_port_device(pt_id);
2424                 }
2425         }
2426
2427         if (hot_plug) {
2428                 ret = rte_dev_event_monitor_stop();
2429                 if (ret) {
2430                         RTE_LOG(ERR, EAL,
2431                                 "fail to stop device event monitor.");
2432                         return;
2433                 }
2434
2435                 ret = rte_dev_event_callback_unregister(NULL,
2436                         dev_event_callback, NULL);
2437                 if (ret < 0) {
2438                         RTE_LOG(ERR, EAL,
2439                                 "fail to unregister device event callback.\n");
2440                         return;
2441                 }
2442
2443                 ret = rte_dev_hotplug_handle_disable();
2444                 if (ret) {
2445                         RTE_LOG(ERR, EAL,
2446                                 "fail to disable hotplug handling.\n");
2447                         return;
2448                 }
2449         }
2450
2451         printf("\nBye...\n");
2452 }
2453
2454 typedef void (*cmd_func_t)(void);
2455 struct pmd_test_command {
2456         const char *cmd_name;
2457         cmd_func_t cmd_func;
2458 };
2459
2460 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2461
2462 /* Check the link status of all ports in up to 9s, and print them finally */
2463 static void
2464 check_all_ports_link_status(uint32_t port_mask)
2465 {
2466 #define CHECK_INTERVAL 100 /* 100ms */
2467 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2468         portid_t portid;
2469         uint8_t count, all_ports_up, print_flag = 0;
2470         struct rte_eth_link link;
2471
2472         printf("Checking link statuses...\n");
2473         fflush(stdout);
2474         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2475                 all_ports_up = 1;
2476                 RTE_ETH_FOREACH_DEV(portid) {
2477                         if ((port_mask & (1 << portid)) == 0)
2478                                 continue;
2479                         memset(&link, 0, sizeof(link));
2480                         rte_eth_link_get_nowait(portid, &link);
2481                         /* print link status if flag set */
2482                         if (print_flag == 1) {
2483                                 if (link.link_status)
2484                                         printf(
2485                                         "Port%d Link Up. speed %u Mbps- %s\n",
2486                                         portid, link.link_speed,
2487                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2488                                         ("full-duplex") : ("half-duplex\n"));
2489                                 else
2490                                         printf("Port %d Link Down\n", portid);
2491                                 continue;
2492                         }
2493                         /* clear all_ports_up flag if any link down */
2494                         if (link.link_status == ETH_LINK_DOWN) {
2495                                 all_ports_up = 0;
2496                                 break;
2497                         }
2498                 }
2499                 /* after finally printing all link status, get out */
2500                 if (print_flag == 1)
2501                         break;
2502
2503                 if (all_ports_up == 0) {
2504                         fflush(stdout);
2505                         rte_delay_ms(CHECK_INTERVAL);
2506                 }
2507
2508                 /* set the print_flag if all ports up or timeout */
2509                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2510                         print_flag = 1;
2511                 }
2512
2513                 if (lsc_interrupt)
2514                         break;
2515         }
2516 }
2517
2518 /*
2519  * This callback is for remove a port for a device. It has limitation because
2520  * it is not for multiple port removal for a device.
2521  * TODO: the device detach invoke will plan to be removed from user side to
2522  * eal. And convert all PMDs to free port resources on ether device closing.
2523  */
2524 static void
2525 rmv_port_callback(void *arg)
2526 {
2527         int need_to_start = 0;
2528         int org_no_link_check = no_link_check;
2529         portid_t port_id = (intptr_t)arg;
2530
2531         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2532
2533         if (!test_done && port_is_forwarding(port_id)) {
2534                 need_to_start = 1;
2535                 stop_packet_forwarding();
2536         }
2537         no_link_check = 1;
2538         stop_port(port_id);
2539         no_link_check = org_no_link_check;
2540         close_port(port_id);
2541         detach_port_device(port_id);
2542         if (need_to_start)
2543                 start_packet_forwarding(0);
2544 }
2545
2546 /* This function is used by the interrupt thread */
2547 static int
2548 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2549                   void *ret_param)
2550 {
2551         RTE_SET_USED(param);
2552         RTE_SET_USED(ret_param);
2553
2554         if (type >= RTE_ETH_EVENT_MAX) {
2555                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2556                         port_id, __func__, type);
2557                 fflush(stderr);
2558         } else if (event_print_mask & (UINT32_C(1) << type)) {
2559                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2560                         eth_event_desc[type]);
2561                 fflush(stdout);
2562         }
2563
2564         switch (type) {
2565         case RTE_ETH_EVENT_NEW:
2566                 ports[port_id].need_setup = 1;
2567                 ports[port_id].port_status = RTE_PORT_HANDLING;
2568                 break;
2569         case RTE_ETH_EVENT_INTR_RMV:
2570                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2571                         break;
2572                 if (rte_eal_alarm_set(100000,
2573                                 rmv_port_callback, (void *)(intptr_t)port_id))
2574                         fprintf(stderr, "Could not set up deferred device removal\n");
2575                 break;
2576         default:
2577                 break;
2578         }
2579         return 0;
2580 }
2581
2582 static int
2583 register_eth_event_callback(void)
2584 {
2585         int ret;
2586         enum rte_eth_event_type event;
2587
2588         for (event = RTE_ETH_EVENT_UNKNOWN;
2589                         event < RTE_ETH_EVENT_MAX; event++) {
2590                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2591                                 event,
2592                                 eth_event_callback,
2593                                 NULL);
2594                 if (ret != 0) {
2595                         TESTPMD_LOG(ERR, "Failed to register callback for "
2596                                         "%s event\n", eth_event_desc[event]);
2597                         return -1;
2598                 }
2599         }
2600
2601         return 0;
2602 }
2603
2604 /* This function is used by the interrupt thread */
2605 static void
2606 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2607                              __rte_unused void *arg)
2608 {
2609         uint16_t port_id;
2610         int ret;
2611
2612         if (type >= RTE_DEV_EVENT_MAX) {
2613                 fprintf(stderr, "%s called upon invalid event %d\n",
2614                         __func__, type);
2615                 fflush(stderr);
2616         }
2617
2618         switch (type) {
2619         case RTE_DEV_EVENT_REMOVE:
2620                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2621                         device_name);
2622                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2623                 if (ret) {
2624                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2625                                 device_name);
2626                         return;
2627                 }
2628                 /*
2629                  * Because the user's callback is invoked in eal interrupt
2630                  * callback, the interrupt callback need to be finished before
2631                  * it can be unregistered when detaching device. So finish
2632                  * callback soon and use a deferred removal to detach device
2633                  * is need. It is a workaround, once the device detaching be
2634                  * moved into the eal in the future, the deferred removal could
2635                  * be deleted.
2636                  */
2637                 if (rte_eal_alarm_set(100000,
2638                                 rmv_port_callback, (void *)(intptr_t)port_id))
2639                         RTE_LOG(ERR, EAL,
2640                                 "Could not set up deferred device removal\n");
2641                 break;
2642         case RTE_DEV_EVENT_ADD:
2643                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2644                         device_name);
2645                 /* TODO: After finish kernel driver binding,
2646                  * begin to attach port.
2647                  */
2648                 break;
2649         default:
2650                 break;
2651         }
2652 }
2653
2654 static int
2655 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2656 {
2657         uint16_t i;
2658         int diag;
2659         uint8_t mapping_found = 0;
2660
2661         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2662                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2663                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2664                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2665                                         tx_queue_stats_mappings[i].queue_id,
2666                                         tx_queue_stats_mappings[i].stats_counter_id);
2667                         if (diag != 0)
2668                                 return diag;
2669                         mapping_found = 1;
2670                 }
2671         }
2672         if (mapping_found)
2673                 port->tx_queue_stats_mapping_enabled = 1;
2674         return 0;
2675 }
2676
2677 static int
2678 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2679 {
2680         uint16_t i;
2681         int diag;
2682         uint8_t mapping_found = 0;
2683
2684         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2685                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2686                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2687                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2688                                         rx_queue_stats_mappings[i].queue_id,
2689                                         rx_queue_stats_mappings[i].stats_counter_id);
2690                         if (diag != 0)
2691                                 return diag;
2692                         mapping_found = 1;
2693                 }
2694         }
2695         if (mapping_found)
2696                 port->rx_queue_stats_mapping_enabled = 1;
2697         return 0;
2698 }
2699
2700 static void
2701 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2702 {
2703         int diag = 0;
2704
2705         diag = set_tx_queue_stats_mapping_registers(pi, port);
2706         if (diag != 0) {
2707                 if (diag == -ENOTSUP) {
2708                         port->tx_queue_stats_mapping_enabled = 0;
2709                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2710                 }
2711                 else
2712                         rte_exit(EXIT_FAILURE,
2713                                         "set_tx_queue_stats_mapping_registers "
2714                                         "failed for port id=%d diag=%d\n",
2715                                         pi, diag);
2716         }
2717
2718         diag = set_rx_queue_stats_mapping_registers(pi, port);
2719         if (diag != 0) {
2720                 if (diag == -ENOTSUP) {
2721                         port->rx_queue_stats_mapping_enabled = 0;
2722                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2723                 }
2724                 else
2725                         rte_exit(EXIT_FAILURE,
2726                                         "set_rx_queue_stats_mapping_registers "
2727                                         "failed for port id=%d diag=%d\n",
2728                                         pi, diag);
2729         }
2730 }
2731
2732 static void
2733 rxtx_port_config(struct rte_port *port)
2734 {
2735         uint16_t qid;
2736
2737         for (qid = 0; qid < nb_rxq; qid++) {
2738                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2739
2740                 /* Check if any Rx parameters have been passed */
2741                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2742                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2743
2744                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2745                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2746
2747                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2748                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2749
2750                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2751                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2752
2753                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2754                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2755
2756                 port->nb_rx_desc[qid] = nb_rxd;
2757         }
2758
2759         for (qid = 0; qid < nb_txq; qid++) {
2760                 port->tx_conf[qid] = port->dev_info.default_txconf;
2761
2762                 /* Check if any Tx parameters have been passed */
2763                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2764                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2765
2766                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2767                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2768
2769                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2770                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2771
2772                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2773                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2774
2775                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2776                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2777
2778                 port->nb_tx_desc[qid] = nb_txd;
2779         }
2780 }
2781
2782 void
2783 init_port_config(void)
2784 {
2785         portid_t pid;
2786         struct rte_port *port;
2787
2788         RTE_ETH_FOREACH_DEV(pid) {
2789                 port = &ports[pid];
2790                 port->dev_conf.fdir_conf = fdir_conf;
2791                 rte_eth_dev_info_get(pid, &port->dev_info);
2792                 if (nb_rxq > 1) {
2793                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2794                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2795                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2796                 } else {
2797                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2798                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2799                 }
2800
2801                 if (port->dcb_flag == 0) {
2802                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2803                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2804                         else
2805                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2806                 }
2807
2808                 rxtx_port_config(port);
2809
2810                 rte_eth_macaddr_get(pid, &port->eth_addr);
2811
2812                 map_port_queue_stats_mapping_registers(pid, port);
2813 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2814                 rte_pmd_ixgbe_bypass_init(pid);
2815 #endif
2816
2817                 if (lsc_interrupt &&
2818                     (rte_eth_devices[pid].data->dev_flags &
2819                      RTE_ETH_DEV_INTR_LSC))
2820                         port->dev_conf.intr_conf.lsc = 1;
2821                 if (rmv_interrupt &&
2822                     (rte_eth_devices[pid].data->dev_flags &
2823                      RTE_ETH_DEV_INTR_RMV))
2824                         port->dev_conf.intr_conf.rmv = 1;
2825         }
2826 }
2827
2828 void set_port_slave_flag(portid_t slave_pid)
2829 {
2830         struct rte_port *port;
2831
2832         port = &ports[slave_pid];
2833         port->slave_flag = 1;
2834 }
2835
2836 void clear_port_slave_flag(portid_t slave_pid)
2837 {
2838         struct rte_port *port;
2839
2840         port = &ports[slave_pid];
2841         port->slave_flag = 0;
2842 }
2843
2844 uint8_t port_is_bonding_slave(portid_t slave_pid)
2845 {
2846         struct rte_port *port;
2847
2848         port = &ports[slave_pid];
2849         if ((rte_eth_devices[slave_pid].data->dev_flags &
2850             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
2851                 return 1;
2852         return 0;
2853 }
2854
2855 const uint16_t vlan_tags[] = {
2856                 0,  1,  2,  3,  4,  5,  6,  7,
2857                 8,  9, 10, 11,  12, 13, 14, 15,
2858                 16, 17, 18, 19, 20, 21, 22, 23,
2859                 24, 25, 26, 27, 28, 29, 30, 31
2860 };
2861
2862 static  int
2863 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
2864                  enum dcb_mode_enable dcb_mode,
2865                  enum rte_eth_nb_tcs num_tcs,
2866                  uint8_t pfc_en)
2867 {
2868         uint8_t i;
2869         int32_t rc;
2870         struct rte_eth_rss_conf rss_conf;
2871
2872         /*
2873          * Builds up the correct configuration for dcb+vt based on the vlan tags array
2874          * given above, and the number of traffic classes available for use.
2875          */
2876         if (dcb_mode == DCB_VT_ENABLED) {
2877                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
2878                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
2879                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2880                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
2881
2882                 /* VMDQ+DCB RX and TX configurations */
2883                 vmdq_rx_conf->enable_default_pool = 0;
2884                 vmdq_rx_conf->default_pool = 0;
2885                 vmdq_rx_conf->nb_queue_pools =
2886                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2887                 vmdq_tx_conf->nb_queue_pools =
2888                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2889
2890                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
2891                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
2892                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
2893                         vmdq_rx_conf->pool_map[i].pools =
2894                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
2895                 }
2896                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2897                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
2898                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
2899                 }
2900
2901                 /* set DCB mode of RX and TX of multiple queues */
2902                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
2903                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
2904         } else {
2905                 struct rte_eth_dcb_rx_conf *rx_conf =
2906                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
2907                 struct rte_eth_dcb_tx_conf *tx_conf =
2908                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
2909
2910                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
2911                 if (rc != 0)
2912                         return rc;
2913
2914                 rx_conf->nb_tcs = num_tcs;
2915                 tx_conf->nb_tcs = num_tcs;
2916
2917                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2918                         rx_conf->dcb_tc[i] = i % num_tcs;
2919                         tx_conf->dcb_tc[i] = i % num_tcs;
2920                 }
2921
2922                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
2923                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
2924                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
2925         }
2926
2927         if (pfc_en)
2928                 eth_conf->dcb_capability_en =
2929                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
2930         else
2931                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
2932
2933         return 0;
2934 }
2935
2936 int
2937 init_port_dcb_config(portid_t pid,
2938                      enum dcb_mode_enable dcb_mode,
2939                      enum rte_eth_nb_tcs num_tcs,
2940                      uint8_t pfc_en)
2941 {
2942         struct rte_eth_conf port_conf;
2943         struct rte_port *rte_port;
2944         int retval;
2945         uint16_t i;
2946
2947         rte_port = &ports[pid];
2948
2949         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
2950         /* Enter DCB configuration status */
2951         dcb_config = 1;
2952
2953         port_conf.rxmode = rte_port->dev_conf.rxmode;
2954         port_conf.txmode = rte_port->dev_conf.txmode;
2955
2956         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
2957         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
2958         if (retval < 0)
2959                 return retval;
2960         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
2961
2962         /* re-configure the device . */
2963         rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
2964
2965         rte_eth_dev_info_get(pid, &rte_port->dev_info);
2966
2967         /* If dev_info.vmdq_pool_base is greater than 0,
2968          * the queue id of vmdq pools is started after pf queues.
2969          */
2970         if (dcb_mode == DCB_VT_ENABLED &&
2971             rte_port->dev_info.vmdq_pool_base > 0) {
2972                 printf("VMDQ_DCB multi-queue mode is nonsensical"
2973                         " for port %d.", pid);
2974                 return -1;
2975         }
2976
2977         /* Assume the ports in testpmd have the same dcb capability
2978          * and has the same number of rxq and txq in dcb mode
2979          */
2980         if (dcb_mode == DCB_VT_ENABLED) {
2981                 if (rte_port->dev_info.max_vfs > 0) {
2982                         nb_rxq = rte_port->dev_info.nb_rx_queues;
2983                         nb_txq = rte_port->dev_info.nb_tx_queues;
2984                 } else {
2985                         nb_rxq = rte_port->dev_info.max_rx_queues;
2986                         nb_txq = rte_port->dev_info.max_tx_queues;
2987                 }
2988         } else {
2989                 /*if vt is disabled, use all pf queues */
2990                 if (rte_port->dev_info.vmdq_pool_base == 0) {
2991                         nb_rxq = rte_port->dev_info.max_rx_queues;
2992                         nb_txq = rte_port->dev_info.max_tx_queues;
2993                 } else {
2994                         nb_rxq = (queueid_t)num_tcs;
2995                         nb_txq = (queueid_t)num_tcs;
2996
2997                 }
2998         }
2999         rx_free_thresh = 64;
3000
3001         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3002
3003         rxtx_port_config(rte_port);
3004         /* VLAN filter */
3005         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3006         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3007                 rx_vft_set(pid, vlan_tags[i], 1);
3008
3009         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
3010         map_port_queue_stats_mapping_registers(pid, rte_port);
3011
3012         rte_port->dcb_flag = 1;
3013
3014         return 0;
3015 }
3016
3017 static void
3018 init_port(void)
3019 {
3020         /* Configuration of Ethernet ports. */
3021         ports = rte_zmalloc("testpmd: ports",
3022                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3023                             RTE_CACHE_LINE_SIZE);
3024         if (ports == NULL) {
3025                 rte_exit(EXIT_FAILURE,
3026                                 "rte_zmalloc(%d struct rte_port) failed\n",
3027                                 RTE_MAX_ETHPORTS);
3028         }
3029
3030         /* Initialize ports NUMA structures */
3031         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3032         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3033         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3034 }
3035
3036 static void
3037 force_quit(void)
3038 {
3039         pmd_test_exit();
3040         prompt_exit();
3041 }
3042
3043 static void
3044 print_stats(void)
3045 {
3046         uint8_t i;
3047         const char clr[] = { 27, '[', '2', 'J', '\0' };
3048         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3049
3050         /* Clear screen and move to top left */
3051         printf("%s%s", clr, top_left);
3052
3053         printf("\nPort statistics ====================================");
3054         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3055                 nic_stats_display(fwd_ports_ids[i]);
3056
3057         fflush(stdout);
3058 }
3059
3060 static void
3061 signal_handler(int signum)
3062 {
3063         if (signum == SIGINT || signum == SIGTERM) {
3064                 printf("\nSignal %d received, preparing to exit...\n",
3065                                 signum);
3066 #ifdef RTE_LIBRTE_PDUMP
3067                 /* uninitialize packet capture framework */
3068                 rte_pdump_uninit();
3069 #endif
3070 #ifdef RTE_LIBRTE_LATENCY_STATS
3071                 rte_latencystats_uninit();
3072 #endif
3073                 force_quit();
3074                 /* Set flag to indicate the force termination. */
3075                 f_quit = 1;
3076                 /* exit with the expected status */
3077                 signal(signum, SIG_DFL);
3078                 kill(getpid(), signum);
3079         }
3080 }
3081
3082 int
3083 main(int argc, char** argv)
3084 {
3085         int diag;
3086         portid_t port_id;
3087         uint16_t count;
3088         int ret;
3089
3090         signal(SIGINT, signal_handler);
3091         signal(SIGTERM, signal_handler);
3092
3093         diag = rte_eal_init(argc, argv);
3094         if (diag < 0)
3095                 rte_panic("Cannot init EAL\n");
3096
3097         testpmd_logtype = rte_log_register("testpmd");
3098         if (testpmd_logtype < 0)
3099                 rte_panic("Cannot register log type");
3100         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3101
3102         ret = register_eth_event_callback();
3103         if (ret != 0)
3104                 rte_panic("Cannot register for ethdev events");
3105
3106 #ifdef RTE_LIBRTE_PDUMP
3107         /* initialize packet capture framework */
3108         rte_pdump_init();
3109 #endif
3110
3111         count = 0;
3112         RTE_ETH_FOREACH_DEV(port_id) {
3113                 ports_ids[count] = port_id;
3114                 count++;
3115         }
3116         nb_ports = (portid_t) count;
3117         if (nb_ports == 0)
3118                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3119
3120         /* allocate port structures, and init them */
3121         init_port();
3122
3123         set_def_fwd_config();
3124         if (nb_lcores == 0)
3125                 rte_panic("Empty set of forwarding logical cores - check the "
3126                           "core mask supplied in the command parameters\n");
3127
3128         /* Bitrate/latency stats disabled by default */
3129 #ifdef RTE_LIBRTE_BITRATE
3130         bitrate_enabled = 0;
3131 #endif
3132 #ifdef RTE_LIBRTE_LATENCY_STATS
3133         latencystats_enabled = 0;
3134 #endif
3135
3136         /* on FreeBSD, mlockall() is disabled by default */
3137 #ifdef RTE_EXEC_ENV_FREEBSD
3138         do_mlockall = 0;
3139 #else
3140         do_mlockall = 1;
3141 #endif
3142
3143         argc -= diag;
3144         argv += diag;
3145         if (argc > 1)
3146                 launch_args_parse(argc, argv);
3147
3148         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3149                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3150                         strerror(errno));
3151         }
3152
3153         if (tx_first && interactive)
3154                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3155                                 "interactive mode.\n");
3156
3157         if (tx_first && lsc_interrupt) {
3158                 printf("Warning: lsc_interrupt needs to be off when "
3159                                 " using tx_first. Disabling.\n");
3160                 lsc_interrupt = 0;
3161         }
3162
3163         if (!nb_rxq && !nb_txq)
3164                 printf("Warning: Either rx or tx queues should be non-zero\n");
3165
3166         if (nb_rxq > 1 && nb_rxq > nb_txq)
3167                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3168                        "but nb_txq=%d will prevent to fully test it.\n",
3169                        nb_rxq, nb_txq);
3170
3171         init_config();
3172
3173         if (hot_plug) {
3174                 ret = rte_dev_hotplug_handle_enable();
3175                 if (ret) {
3176                         RTE_LOG(ERR, EAL,
3177                                 "fail to enable hotplug handling.");
3178                         return -1;
3179                 }
3180
3181                 ret = rte_dev_event_monitor_start();
3182                 if (ret) {
3183                         RTE_LOG(ERR, EAL,
3184                                 "fail to start device event monitoring.");
3185                         return -1;
3186                 }
3187
3188                 ret = rte_dev_event_callback_register(NULL,
3189                         dev_event_callback, NULL);
3190                 if (ret) {
3191                         RTE_LOG(ERR, EAL,
3192                                 "fail  to register device event callback\n");
3193                         return -1;
3194                 }
3195         }
3196
3197         if (start_port(RTE_PORT_ALL) != 0)
3198                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3199
3200         /* set all ports to promiscuous mode by default */
3201         RTE_ETH_FOREACH_DEV(port_id)
3202                 rte_eth_promiscuous_enable(port_id);
3203
3204         /* Init metrics library */
3205         rte_metrics_init(rte_socket_id());
3206
3207 #ifdef RTE_LIBRTE_LATENCY_STATS
3208         if (latencystats_enabled != 0) {
3209                 int ret = rte_latencystats_init(1, NULL);
3210                 if (ret)
3211                         printf("Warning: latencystats init()"
3212                                 " returned error %d\n", ret);
3213                 printf("Latencystats running on lcore %d\n",
3214                         latencystats_lcore_id);
3215         }
3216 #endif
3217
3218         /* Setup bitrate stats */
3219 #ifdef RTE_LIBRTE_BITRATE
3220         if (bitrate_enabled != 0) {
3221                 bitrate_data = rte_stats_bitrate_create();
3222                 if (bitrate_data == NULL)
3223                         rte_exit(EXIT_FAILURE,
3224                                 "Could not allocate bitrate data.\n");
3225                 rte_stats_bitrate_reg(bitrate_data);
3226         }
3227 #endif
3228
3229 #ifdef RTE_LIBRTE_CMDLINE
3230         if (strlen(cmdline_filename) != 0)
3231                 cmdline_read_from_file(cmdline_filename);
3232
3233         if (interactive == 1) {
3234                 if (auto_start) {
3235                         printf("Start automatic packet forwarding\n");
3236                         start_packet_forwarding(0);
3237                 }
3238                 prompt();
3239                 pmd_test_exit();
3240         } else
3241 #endif
3242         {
3243                 char c;
3244                 int rc;
3245
3246                 f_quit = 0;
3247
3248                 printf("No commandline core given, start packet forwarding\n");
3249                 start_packet_forwarding(tx_first);
3250                 if (stats_period != 0) {
3251                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3252                         uint64_t timer_period;
3253
3254                         /* Convert to number of cycles */
3255                         timer_period = stats_period * rte_get_timer_hz();
3256
3257                         while (f_quit == 0) {
3258                                 cur_time = rte_get_timer_cycles();
3259                                 diff_time += cur_time - prev_time;
3260
3261                                 if (diff_time >= timer_period) {
3262                                         print_stats();
3263                                         /* Reset the timer */
3264                                         diff_time = 0;
3265                                 }
3266                                 /* Sleep to avoid unnecessary checks */
3267                                 prev_time = cur_time;
3268                                 sleep(1);
3269                         }
3270                 }
3271
3272                 printf("Press enter to exit\n");
3273                 rc = read(0, &c, 1);
3274                 pmd_test_exit();
3275                 if (rc < 0)
3276                         return 1;
3277         }
3278
3279         return 0;
3280 }