app/test-pmd/testpmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdarg.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <signal.h>
  38 #include <string.h>
  39 #include <time.h>
  40 #include <fcntl.h>
  41 #include <sys/types.h>
  42 #include <errno.h>
  43
  44 #include <sys/queue.h>
  45 #include <sys/stat.h>
  46
  47 #include <stdint.h>
  48 #include <unistd.h>
  49 #include <inttypes.h>
  50
  51 #include <rte_common.h>
  52 #include <rte_errno.h>
  53 #include <rte_byteorder.h>
  54 #include <rte_log.h>
  55 #include <rte_debug.h>
  56 #include <rte_cycles.h>
  57 #include <rte_memory.h>
  58 #include <rte_memcpy.h>
  59 #include <rte_memzone.h>
  60 #include <rte_launch.h>
  61 #include <rte_eal.h>
  62 #include <rte_alarm.h>
  63 #include <rte_per_lcore.h>
  64 #include <rte_lcore.h>
  65 #include <rte_atomic.h>
  66 #include <rte_branch_prediction.h>
  67 #include <rte_mempool.h>
  68 #include <rte_malloc.h>
  69 #include <rte_mbuf.h>
  70 #include <rte_interrupts.h>
  71 #include <rte_pci.h>
  72 #include <rte_ether.h>
  73 #include <rte_ethdev.h>
  74 #include <rte_dev.h>
  75 #include <rte_string_fns.h>
  76 #ifdef RTE_LIBRTE_IXGBE_PMD
  77 #include <rte_pmd_ixgbe.h>
  78 #endif
  79 #ifdef RTE_LIBRTE_PMD_XENVIRT
  80 #include <rte_eth_xenvirt.h>
  81 #endif
  82 #ifdef RTE_LIBRTE_PDUMP
  83 #include <rte_pdump.h>
  84 #endif
  85 #include <rte_flow.h>
  86 #include <rte_metrics.h>
  87 #ifdef RTE_LIBRTE_BITRATE
  88 #include <rte_bitrate.h>
  89 #endif
  90 #ifdef RTE_LIBRTE_LATENCY_STATS
  91 #include <rte_latencystats.h>
  92 #endif
  93 #include <rte_gro.h>
  94
  95 #include "testpmd.h"
  96
  97 uint16_t verbose_level = 0; /**< Silent by default. */
  98
  99 /* use master core for command line ? */
 100 uint8_t interactive = 0;
 101 uint8_t auto_start = 0;
 102 uint8_t tx_first;
 103 char cmdline_filename[PATH_MAX] = {0};
 104
 105 /*
 106  * NUMA support configuration.
 107  * When set, the NUMA support attempts to dispatch the allocation of the
 108  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 109  * probed ports among the CPU sockets 0 and 1.
 110  * Otherwise, all memory is allocated from CPU socket 0.
 111  */
 112 uint8_t numa_support = 1; /**< numa enabled by default */
 113
 114 /*
 115  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 116  * not configured.
 117  */
 118 uint8_t socket_num = UMA_NO_CONFIG;
 119
 120 /*
 121  * Use ANONYMOUS mapped memory (might be not physically continuous) for mbufs.
 122  */
 123 uint8_t mp_anon = 0;
 124
 125 /*
 126  * Record the Ethernet address of peer target ports to which packets are
 127  * forwarded.
 128  * Must be instantiated with the ethernet addresses of peer traffic generator
 129  * ports.
 130  */
 131 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 132 portid_t nb_peer_eth_addrs = 0;
 133
 134 /*
 135  * Probed Target Environment.
 136  */
 137 struct rte_port *ports;        /**< For all probed ethernet ports. */
 138 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 139 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 140 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 141
 142 /*
 143  * Test Forwarding Configuration.
 144  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 145  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 146  */
 147 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 148 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 149 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 150 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 151
 152 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 153 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 154
 155 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 156 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 157
 158 /*
 159  * Forwarding engines.
 160  */
 161 struct fwd_engine * fwd_engines[] = {
 162         &io_fwd_engine,
 163         &mac_fwd_engine,
 164         &mac_swap_engine,
 165         &flow_gen_engine,
 166         &rx_only_engine,
 167         &tx_only_engine,
 168         &csum_fwd_engine,
 169         &icmp_echo_engine,
 170 #ifdef RTE_LIBRTE_IEEE1588
 171         &ieee1588_fwd_engine,
 172 #endif
 173         NULL,
 174 };
 175
 176 struct fwd_config cur_fwd_config;
 177 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 178 uint32_t retry_enabled;
 179 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 180 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 181
 182 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 183 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 184                                       * specified on command-line. */
 185 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 186 /*
 187  * Configuration of packet segments used by the "txonly" processing engine.
 188  */
 189 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 190 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 191         TXONLY_DEF_PACKET_LEN,
 192 };
 193 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 194
 195 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 196 /**< Split policy for packets to TX. */
 197
 198 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 199 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 200
 201 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 202 uint8_t dcb_config = 0;
 203
 204 /* Whether the dcb is in testing status */
 205 uint8_t dcb_test = 0;
 206
 207 /*
 208  * Configurable number of RX/TX queues.
 209  */
 210 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 211 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 212
 213 /*
 214  * Configurable number of RX/TX ring descriptors.
 215  */
 216 #define RTE_TEST_RX_DESC_DEFAULT 128
 217 #define RTE_TEST_TX_DESC_DEFAULT 512
 218 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 219 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 220
 221 #define RTE_PMD_PARAM_UNSET -1
 222 /*
 223  * Configurable values of RX and TX ring threshold registers.
 224  */
 225
 226 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 227 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 228 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 229
 230 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 231 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 232 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 233
 234 /*
 235  * Configurable value of RX free threshold.
 236  */
 237 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 238
 239 /*
 240  * Configurable value of RX drop enable.
 241  */
 242 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 243
 244 /*
 245  * Configurable value of TX free threshold.
 246  */
 247 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 248
 249 /*
 250  * Configurable value of TX RS bit threshold.
 251  */
 252 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 253
 254 /*
 255  * Configurable value of TX queue flags.
 256  */
 257 int32_t txq_flags = RTE_PMD_PARAM_UNSET;
 258
 259 /*
 260  * Receive Side Scaling (RSS) configuration.
 261  */
 262 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 263
 264 /*
 265  * Port topology configuration
 266  */
 267 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 268
 269 /*
 270  * Avoids to flush all the RX streams before starts forwarding.
 271  */
 272 uint8_t no_flush_rx = 0; /* flush by default */
 273
 274 /*
 275  * Flow API isolated mode.
 276  */
 277 uint8_t flow_isolate_all;
 278
 279 /*
 280  * Avoids to check link status when starting/stopping a port.
 281  */
 282 uint8_t no_link_check = 0; /* check by default */
 283
 284 /*
 285  * Enable link status change notification
 286  */
 287 uint8_t lsc_interrupt = 1; /* enabled by default */
 288
 289 /*
 290  * Enable device removal notification.
 291  */
 292 uint8_t rmv_interrupt = 1; /* enabled by default */
 293
 294 /*
 295  * Display or mask ether events
 296  * Default to all events except VF_MBOX
 297  */
 298 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 299                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 300                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 301                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 302                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 303                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 304
 305 /*
 306  * NIC bypass mode configuration options.
 307  */
 308
 309 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 310 /* The NIC bypass watchdog timeout. */
 311 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 312 #endif
 313
 314
 315 #ifdef RTE_LIBRTE_LATENCY_STATS
 316
 317 /*
 318  * Set when latency stats is enabled in the commandline
 319  */
 320 uint8_t latencystats_enabled;
 321
 322 /*
 323  * Lcore ID to serive latency statistics.
 324  */
 325 lcoreid_t latencystats_lcore_id = -1;
 326
 327 #endif
 328
 329 /*
 330  * Ethernet device configuration.
 331  */
 332 struct rte_eth_rxmode rx_mode = {
 333         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 334         .split_hdr_size = 0,
 335         .header_split   = 0, /**< Header Split disabled. */
 336         .hw_ip_checksum = 0, /**< IP checksum offload disabled. */
 337         .hw_vlan_filter = 1, /**< VLAN filtering enabled. */
 338         .hw_vlan_strip  = 1, /**< VLAN strip enabled. */
 339         .hw_vlan_extend = 0, /**< Extended VLAN disabled. */
 340         .jumbo_frame    = 0, /**< Jumbo Frame Support disabled. */
 341         .hw_strip_crc   = 1, /**< CRC stripping by hardware enabled. */
 342 };
 343
 344 struct rte_fdir_conf fdir_conf = {
 345         .mode = RTE_FDIR_MODE_NONE,
 346         .pballoc = RTE_FDIR_PBALLOC_64K,
 347         .status = RTE_FDIR_REPORT_STATUS,
 348         .mask = {
 349                 .vlan_tci_mask = 0x0,
 350                 .ipv4_mask     = {
 351                         .src_ip = 0xFFFFFFFF,
 352                         .dst_ip = 0xFFFFFFFF,
 353                 },
 354                 .ipv6_mask     = {
 355                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 356                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 357                 },
 358                 .src_port_mask = 0xFFFF,
 359                 .dst_port_mask = 0xFFFF,
 360                 .mac_addr_byte_mask = 0xFF,
 361                 .tunnel_type_mask = 1,
 362                 .tunnel_id_mask = 0xFFFFFFFF,
 363         },
 364         .drop_queue = 127,
 365 };
 366
 367 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 368
 369 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 370 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 371
 372 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 373 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 374
 375 uint16_t nb_tx_queue_stats_mappings = 0;
 376 uint16_t nb_rx_queue_stats_mappings = 0;
 377
 378 unsigned int num_sockets = 0;
 379 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 380
 381 #ifdef RTE_LIBRTE_BITRATE
 382 /* Bitrate statistics */
 383 struct rte_stats_bitrates *bitrate_data;
 384 lcoreid_t bitrate_lcore_id;
 385 uint8_t bitrate_enabled;
 386 #endif
 387
 388 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 389
 390 /* Forward function declarations */
 391 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port);
 392 static void check_all_ports_link_status(uint32_t port_mask);
 393 static int eth_event_callback(uint8_t port_id,
 394                               enum rte_eth_event_type type,
 395                               void *param, void *ret_param);
 396
 397 /*
 398  * Check if all the ports are started.
 399  * If yes, return positive value. If not, return zero.
 400  */
 401 static int all_ports_started(void);
 402
 403 /*
 404  * Helper function to check if socket is already discovered.
 405  * If yes, return positive value. If not, return zero.
 406  */
 407 int
 408 new_socket_id(unsigned int socket_id)
 409 {
 410         unsigned int i;
 411
 412         for (i = 0; i < num_sockets; i++) {
 413                 if (socket_ids[i] == socket_id)
 414                         return 0;
 415         }
 416         return 1;
 417 }
 418
 419 /*
 420  * Setup default configuration.
 421  */
 422 static void
 423 set_default_fwd_lcores_config(void)
 424 {
 425         unsigned int i;
 426         unsigned int nb_lc;
 427         unsigned int sock_num;
 428
 429         nb_lc = 0;
 430         for (i = 0; i < RTE_MAX_LCORE; i++) {
 431                 sock_num = rte_lcore_to_socket_id(i);
 432                 if (new_socket_id(sock_num)) {
 433                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 434                                 rte_exit(EXIT_FAILURE,
 435                                          "Total sockets greater than %u\n",
 436                                          RTE_MAX_NUMA_NODES);
 437                         }
 438                         socket_ids[num_sockets++] = sock_num;
 439                 }
 440                 if (!rte_lcore_is_enabled(i))
 441                         continue;
 442                 if (i == rte_get_master_lcore())
 443                         continue;
 444                 fwd_lcores_cpuids[nb_lc++] = i;
 445         }
 446         nb_lcores = (lcoreid_t) nb_lc;
 447         nb_cfg_lcores = nb_lcores;
 448         nb_fwd_lcores = 1;
 449 }
 450
 451 static void
 452 set_def_peer_eth_addrs(void)
 453 {
 454         portid_t i;
 455
 456         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 457                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 458                 peer_eth_addrs[i].addr_bytes[5] = i;
 459         }
 460 }
 461
 462 static void
 463 set_default_fwd_ports_config(void)
 464 {
 465         portid_t pt_id;
 466
 467         for (pt_id = 0; pt_id < nb_ports; pt_id++)
 468                 fwd_ports_ids[pt_id] = pt_id;
 469
 470         nb_cfg_ports = nb_ports;
 471         nb_fwd_ports = nb_ports;
 472 }
 473
 474 void
 475 set_def_fwd_config(void)
 476 {
 477         set_default_fwd_lcores_config();
 478         set_def_peer_eth_addrs();
 479         set_default_fwd_ports_config();
 480 }
 481
 482 /*
 483  * Configuration initialisation done once at init time.
 484  */
 485 static void
 486 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 487                  unsigned int socket_id)
 488 {
 489         char pool_name[RTE_MEMPOOL_NAMESIZE];
 490         struct rte_mempool *rte_mp = NULL;
 491         uint32_t mb_size;
 492
 493         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 494         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 495
 496         RTE_LOG(INFO, USER1,
 497                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 498                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 499
 500 #ifdef RTE_LIBRTE_PMD_XENVIRT
 501         rte_mp = rte_mempool_gntalloc_create(pool_name, nb_mbuf, mb_size,
 502                 (unsigned) mb_mempool_cache,
 503                 sizeof(struct rte_pktmbuf_pool_private),
 504                 rte_pktmbuf_pool_init, NULL,
 505                 rte_pktmbuf_init, NULL,
 506                 socket_id, 0);
 507 #endif
 508
 509         /* if the former XEN allocation failed fall back to normal allocation */
 510         if (rte_mp == NULL) {
 511                 if (mp_anon != 0) {
 512                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 513                                 mb_size, (unsigned) mb_mempool_cache,
 514                                 sizeof(struct rte_pktmbuf_pool_private),
 515                                 socket_id, 0);
 516                         if (rte_mp == NULL)
 517                                 goto err;
 518
 519                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 520                                 rte_mempool_free(rte_mp);
 521                                 rte_mp = NULL;
 522                                 goto err;
 523                         }
 524                         rte_pktmbuf_pool_init(rte_mp, NULL);
 525                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 526                 } else {
 527                         /* wrapper to rte_mempool_create() */
 528                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 529                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 530                 }
 531         }
 532
 533 err:
 534         if (rte_mp == NULL) {
 535                 rte_exit(EXIT_FAILURE,
 536                         "Creation of mbuf pool for socket %u failed: %s\n",
 537                         socket_id, rte_strerror(rte_errno));
 538         } else if (verbose_level > 0) {
 539                 rte_mempool_dump(stdout, rte_mp);
 540         }
 541 }
 542
 543 /*
 544  * Check given socket id is valid or not with NUMA mode,
 545  * if valid, return 0, else return -1
 546  */
 547 static int
 548 check_socket_id(const unsigned int socket_id)
 549 {
 550         static int warning_once = 0;
 551
 552         if (new_socket_id(socket_id)) {
 553                 if (!warning_once && numa_support)
 554                         printf("Warning: NUMA should be configured manually by"
 555                                " using --port-numa-config and"
 556                                " --ring-numa-config parameters along with"
 557                                " --numa.\n");
 558                 warning_once = 1;
 559                 return -1;
 560         }
 561         return 0;
 562 }
 563
 564 static void
 565 init_config(void)
 566 {
 567         portid_t pid;
 568         struct rte_port *port;
 569         struct rte_mempool *mbp;
 570         unsigned int nb_mbuf_per_pool;
 571         lcoreid_t  lc_id;
 572         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
 573
 574         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
 575
 576         if (numa_support) {
 577                 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
 578                 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
 579                 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
 580         }
 581
 582         /* Configuration of logical cores. */
 583         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
 584                                 sizeof(struct fwd_lcore *) * nb_lcores,
 585                                 RTE_CACHE_LINE_SIZE);
 586         if (fwd_lcores == NULL) {
 587                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
 588                                                         "failed\n", nb_lcores);
 589         }
 590         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 591                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
 592                                                sizeof(struct fwd_lcore),
 593                                                RTE_CACHE_LINE_SIZE);
 594                 if (fwd_lcores[lc_id] == NULL) {
 595                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
 596                                                                 "failed\n");
 597                 }
 598                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
 599         }
 600
 601         RTE_ETH_FOREACH_DEV(pid) {
 602                 port = &ports[pid];
 603                 rte_eth_dev_info_get(pid, &port->dev_info);
 604
 605                 if (numa_support) {
 606                         if (port_numa[pid] != NUMA_NO_CONFIG)
 607                                 port_per_socket[port_numa[pid]]++;
 608                         else {
 609                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
 610
 611                                 /* if socket_id is invalid, set to 0 */
 612                                 if (check_socket_id(socket_id) < 0)
 613                                         socket_id = 0;
 614                                 port_per_socket[socket_id]++;
 615                         }
 616                 }
 617
 618                 /* set flag to initialize port/queue */
 619                 port->need_reconfig = 1;
 620                 port->need_reconfig_queues = 1;
 621         }
 622
 623         /*
 624          * Create pools of mbuf.
 625          * If NUMA support is disabled, create a single pool of mbuf in
 626          * socket 0 memory by default.
 627          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
 628          *
 629          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
 630          * nb_txd can be configured at run time.
 631          */
 632         if (param_total_num_mbufs)
 633                 nb_mbuf_per_pool = param_total_num_mbufs;
 634         else {
 635                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
 636                         (nb_lcores * mb_mempool_cache) +
 637                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
 638                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
 639         }
 640
 641         if (numa_support) {
 642                 uint8_t i;
 643
 644                 for (i = 0; i < num_sockets; i++)
 645                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
 646                                          socket_ids[i]);
 647         } else {
 648                 if (socket_num == UMA_NO_CONFIG)
 649                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0);
 650                 else
 651                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
 652                                                  socket_num);
 653         }
 654
 655         init_port_config();
 656
 657         /*
 658          * Records which Mbuf pool to use by each logical core, if needed.
 659          */
 660         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 661                 mbp = mbuf_pool_find(
 662                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
 663
 664                 if (mbp == NULL)
 665                         mbp = mbuf_pool_find(0);
 666                 fwd_lcores[lc_id]->mbp = mbp;
 667         }
 668
 669         /* Configuration of packet forwarding streams. */
 670         if (init_fwd_streams() < 0)
 671                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 672
 673         fwd_config_setup();
 674 }
 675
 676
 677 void
 678 reconfig(portid_t new_port_id, unsigned socket_id)
 679 {
 680         struct rte_port *port;
 681
 682         /* Reconfiguration of Ethernet ports. */
 683         port = &ports[new_port_id];
 684         rte_eth_dev_info_get(new_port_id, &port->dev_info);
 685
 686         /* set flag to initialize port/queue */
 687         port->need_reconfig = 1;
 688         port->need_reconfig_queues = 1;
 689         port->socket_id = socket_id;
 690
 691         init_port_config();
 692 }
 693
 694
 695 int
 696 init_fwd_streams(void)
 697 {
 698         portid_t pid;
 699         struct rte_port *port;
 700         streamid_t sm_id, nb_fwd_streams_new;
 701         queueid_t q;
 702
 703         /* set socket id according to numa or not */
 704         RTE_ETH_FOREACH_DEV(pid) {
 705                 port = &ports[pid];
 706                 if (nb_rxq > port->dev_info.max_rx_queues) {
 707                         printf("Fail: nb_rxq(%d) is greater than "
 708                                 "max_rx_queues(%d)\n", nb_rxq,
 709                                 port->dev_info.max_rx_queues);
 710                         return -1;
 711                 }
 712                 if (nb_txq > port->dev_info.max_tx_queues) {
 713                         printf("Fail: nb_txq(%d) is greater than "
 714                                 "max_tx_queues(%d)\n", nb_txq,
 715                                 port->dev_info.max_tx_queues);
 716                         return -1;
 717                 }
 718                 if (numa_support) {
 719                         if (port_numa[pid] != NUMA_NO_CONFIG)
 720                                 port->socket_id = port_numa[pid];
 721                         else {
 722                                 port->socket_id = rte_eth_dev_socket_id(pid);
 723
 724                                 /* if socket_id is invalid, set to 0 */
 725                                 if (check_socket_id(port->socket_id) < 0)
 726                                         port->socket_id = 0;
 727                         }
 728                 }
 729                 else {
 730                         if (socket_num == UMA_NO_CONFIG)
 731                                 port->socket_id = 0;
 732                         else
 733                                 port->socket_id = socket_num;
 734                 }
 735         }
 736
 737         q = RTE_MAX(nb_rxq, nb_txq);
 738         if (q == 0) {
 739                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
 740                 return -1;
 741         }
 742         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
 743         if (nb_fwd_streams_new == nb_fwd_streams)
 744                 return 0;
 745         /* clear the old */
 746         if (fwd_streams != NULL) {
 747                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 748                         if (fwd_streams[sm_id] == NULL)
 749                                 continue;
 750                         rte_free(fwd_streams[sm_id]);
 751                         fwd_streams[sm_id] = NULL;
 752                 }
 753                 rte_free(fwd_streams);
 754                 fwd_streams = NULL;
 755         }
 756
 757         /* init new */
 758         nb_fwd_streams = nb_fwd_streams_new;
 759         fwd_streams = rte_zmalloc("testpmd: fwd_streams",
 760                 sizeof(struct fwd_stream *) * nb_fwd_streams, RTE_CACHE_LINE_SIZE);
 761         if (fwd_streams == NULL)
 762                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_stream *)) "
 763                                                 "failed\n", nb_fwd_streams);
 764
 765         for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 766                 fwd_streams[sm_id] = rte_zmalloc("testpmd: struct fwd_stream",
 767                                 sizeof(struct fwd_stream), RTE_CACHE_LINE_SIZE);
 768                 if (fwd_streams[sm_id] == NULL)
 769                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_stream)"
 770                                                                 " failed\n");
 771         }
 772
 773         return 0;
 774 }
 775
 776 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 777 static void
 778 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
 779 {
 780         unsigned int total_burst;
 781         unsigned int nb_burst;
 782         unsigned int burst_stats[3];
 783         uint16_t pktnb_stats[3];
 784         uint16_t nb_pkt;
 785         int burst_percent[3];
 786
 787         /*
 788          * First compute the total number of packet bursts and the
 789          * two highest numbers of bursts of the same number of packets.
 790          */
 791         total_burst = 0;
 792         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
 793         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
 794         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
 795                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
 796                 if (nb_burst == 0)
 797                         continue;
 798                 total_burst += nb_burst;
 799                 if (nb_burst > burst_stats[0]) {
 800                         burst_stats[1] = burst_stats[0];
 801                         pktnb_stats[1] = pktnb_stats[0];
 802                         burst_stats[0] = nb_burst;
 803                         pktnb_stats[0] = nb_pkt;
 804                 }
 805         }
 806         if (total_burst == 0)
 807                 return;
 808         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
 809         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
 810                burst_percent[0], (int) pktnb_stats[0]);
 811         if (burst_stats[0] == total_burst) {
 812                 printf("]\n");
 813                 return;
 814         }
 815         if (burst_stats[0] + burst_stats[1] == total_burst) {
 816                 printf(" + %d%% of %d pkts]\n",
 817                        100 - burst_percent[0], pktnb_stats[1]);
 818                 return;
 819         }
 820         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
 821         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
 822         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
 823                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
 824                 return;
 825         }
 826         printf(" + %d%% of %d pkts + %d%% of others]\n",
 827                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
 828 }
 829 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
 830
 831 static void
 832 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats)
 833 {
 834         struct rte_port *port;
 835         uint8_t i;
 836
 837         static const char *fwd_stats_border = "----------------------";
 838
 839         port = &ports[port_id];
 840         printf("\n  %s Forward statistics for port %-2d %s\n",
 841                fwd_stats_border, port_id, fwd_stats_border);
 842
 843         if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) {
 844                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
 845                        "%-"PRIu64"\n",
 846                        stats->ipackets, stats->imissed,
 847                        (uint64_t) (stats->ipackets + stats->imissed));
 848
 849                 if (cur_fwd_eng == &csum_fwd_engine)
 850                         printf("  Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n",
 851                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 852                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
 853                         printf("  RX-error: %-"PRIu64"\n",  stats->ierrors);
 854                         printf("  RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf);
 855                 }
 856
 857                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
 858                        "%-"PRIu64"\n",
 859                        stats->opackets, port->tx_dropped,
 860                        (uint64_t) (stats->opackets + port->tx_dropped));
 861         }
 862         else {
 863                 printf("  RX-packets:             %14"PRIu64"    RX-dropped:%14"PRIu64"    RX-total:"
 864                        "%14"PRIu64"\n",
 865                        stats->ipackets, stats->imissed,
 866                        (uint64_t) (stats->ipackets + stats->imissed));
 867
 868                 if (cur_fwd_eng == &csum_fwd_engine)
 869                         printf("  Bad-ipcsum:%14"PRIu64"    Bad-l4csum:%14"PRIu64"\n",
 870                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 871                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
 872                         printf("  RX-error:%"PRIu64"\n", stats->ierrors);
 873                         printf("  RX-nombufs:             %14"PRIu64"\n",
 874                                stats->rx_nombuf);
 875                 }
 876
 877                 printf("  TX-packets:             %14"PRIu64"    TX-dropped:%14"PRIu64"    TX-total:"
 878                        "%14"PRIu64"\n",
 879                        stats->opackets, port->tx_dropped,
 880                        (uint64_t) (stats->opackets + port->tx_dropped));
 881         }
 882
 883 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 884         if (port->rx_stream)
 885                 pkt_burst_stats_display("RX",
 886                         &port->rx_stream->rx_burst_stats);
 887         if (port->tx_stream)
 888                 pkt_burst_stats_display("TX",
 889                         &port->tx_stream->tx_burst_stats);
 890 #endif
 891
 892         if (port->rx_queue_stats_mapping_enabled) {
 893                 printf("\n");
 894                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 895                         printf("  Stats reg %2d RX-packets:%14"PRIu64
 896                                "     RX-errors:%14"PRIu64
 897                                "    RX-bytes:%14"PRIu64"\n",
 898                                i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]);
 899                 }
 900                 printf("\n");
 901         }
 902         if (port->tx_queue_stats_mapping_enabled) {
 903                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 904                         printf("  Stats reg %2d TX-packets:%14"PRIu64
 905                                "                                 TX-bytes:%14"PRIu64"\n",
 906                                i, stats->q_opackets[i], stats->q_obytes[i]);
 907                 }
 908         }
 909
 910         printf("  %s--------------------------------%s\n",
 911                fwd_stats_border, fwd_stats_border);
 912 }
 913
 914 static void
 915 fwd_stream_stats_display(streamid_t stream_id)
 916 {
 917         struct fwd_stream *fs;
 918         static const char *fwd_top_stats_border = "-------";
 919
 920         fs = fwd_streams[stream_id];
 921         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
 922             (fs->fwd_dropped == 0))
 923                 return;
 924         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
 925                "TX Port=%2d/Queue=%2d %s\n",
 926                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
 927                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
 928         printf("  RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u",
 929                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
 930
 931         /* if checksum mode */
 932         if (cur_fwd_eng == &csum_fwd_engine) {
 933                printf("  RX- bad IP checksum: %-14u  Rx- bad L4 checksum: "
 934                         "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum);
 935         }
 936
 937 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 938         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
 939         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
 940 #endif
 941 }
 942
 943 static void
 944 flush_fwd_rx_queues(void)
 945 {
 946         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 947         portid_t  rxp;
 948         portid_t port_id;
 949         queueid_t rxq;
 950         uint16_t  nb_rx;
 951         uint16_t  i;
 952         uint8_t   j;
 953         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
 954         uint64_t timer_period;
 955
 956         /* convert to number of cycles */
 957         timer_period = rte_get_timer_hz(); /* 1 second timeout */
 958
 959         for (j = 0; j < 2; j++) {
 960                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
 961                         for (rxq = 0; rxq < nb_rxq; rxq++) {
 962                                 port_id = fwd_ports_ids[rxp];
 963                                 /**
 964                                 * testpmd can stuck in the below do while loop
 965                                 * if rte_eth_rx_burst() always returns nonzero
 966                                 * packets. So timer is added to exit this loop
 967                                 * after 1sec timer expiry.
 968                                 */
 969                                 prev_tsc = rte_rdtsc();
 970                                 do {
 971                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
 972                                                 pkts_burst, MAX_PKT_BURST);
 973                                         for (i = 0; i < nb_rx; i++)
 974                                                 rte_pktmbuf_free(pkts_burst[i]);
 975
 976                                         cur_tsc = rte_rdtsc();
 977                                         diff_tsc = cur_tsc - prev_tsc;
 978                                         timer_tsc += diff_tsc;
 979                                 } while ((nb_rx > 0) &&
 980                                         (timer_tsc < timer_period));
 981                                 timer_tsc = 0;
 982                         }
 983                 }
 984                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
 985         }
 986 }
 987
 988 static void
 989 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
 990 {
 991         struct fwd_stream **fsm;
 992         streamid_t nb_fs;
 993         streamid_t sm_id;
 994 #ifdef RTE_LIBRTE_BITRATE
 995         uint64_t tics_per_1sec;
 996         uint64_t tics_datum;
 997         uint64_t tics_current;
 998         uint8_t idx_port, cnt_ports;
 999
1000         cnt_ports = rte_eth_dev_count();
1001         tics_datum = rte_rdtsc();
1002         tics_per_1sec = rte_get_timer_hz();
1003 #endif
1004         fsm = &fwd_streams[fc->stream_idx];
1005         nb_fs = fc->stream_nb;
1006         do {
1007                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1008                         (*pkt_fwd)(fsm[sm_id]);
1009 #ifdef RTE_LIBRTE_BITRATE
1010                 if (bitrate_enabled != 0 &&
1011                                 bitrate_lcore_id == rte_lcore_id()) {
1012                         tics_current = rte_rdtsc();
1013                         if (tics_current - tics_datum >= tics_per_1sec) {
1014                                 /* Periodic bitrate calculation */
1015                                 for (idx_port = 0;
1016                                                 idx_port < cnt_ports;
1017                                                 idx_port++)
1018                                         rte_stats_bitrate_calc(bitrate_data,
1019                                                 idx_port);
1020                                 tics_datum = tics_current;
1021                         }
1022                 }
1023 #endif
1024 #ifdef RTE_LIBRTE_LATENCY_STATS
1025                 if (latencystats_enabled != 0 &&
1026                                 latencystats_lcore_id == rte_lcore_id())
1027                         rte_latencystats_update();
1028 #endif
1029
1030         } while (! fc->stopped);
1031 }
1032
1033 static int
1034 start_pkt_forward_on_core(void *fwd_arg)
1035 {
1036         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1037                              cur_fwd_config.fwd_eng->packet_fwd);
1038         return 0;
1039 }
1040
1041 /*
1042  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1043  * Used to start communication flows in network loopback test configurations.
1044  */
1045 static int
1046 run_one_txonly_burst_on_core(void *fwd_arg)
1047 {
1048         struct fwd_lcore *fwd_lc;
1049         struct fwd_lcore tmp_lcore;
1050
1051         fwd_lc = (struct fwd_lcore *) fwd_arg;
1052         tmp_lcore = *fwd_lc;
1053         tmp_lcore.stopped = 1;
1054         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1055         return 0;
1056 }
1057
1058 /*
1059  * Launch packet forwarding:
1060  *     - Setup per-port forwarding context.
1061  *     - launch logical cores with their forwarding configuration.
1062  */
1063 static void
1064 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1065 {
1066         port_fwd_begin_t port_fwd_begin;
1067         unsigned int i;
1068         unsigned int lc_id;
1069         int diag;
1070
1071         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1072         if (port_fwd_begin != NULL) {
1073                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1074                         (*port_fwd_begin)(fwd_ports_ids[i]);
1075         }
1076         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1077                 lc_id = fwd_lcores_cpuids[i];
1078                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1079                         fwd_lcores[i]->stopped = 0;
1080                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1081                                                      fwd_lcores[i], lc_id);
1082                         if (diag != 0)
1083                                 printf("launch lcore %u failed - diag=%d\n",
1084                                        lc_id, diag);
1085                 }
1086         }
1087 }
1088
1089 /*
1090  * Launch packet forwarding configuration.
1091  */
1092 void
1093 start_packet_forwarding(int with_tx_first)
1094 {
1095         port_fwd_begin_t port_fwd_begin;
1096         port_fwd_end_t  port_fwd_end;
1097         struct rte_port *port;
1098         unsigned int i;
1099         portid_t   pt_id;
1100         streamid_t sm_id;
1101
1102         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1103                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1104
1105         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1106                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1107
1108         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1109                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1110                 (!nb_rxq || !nb_txq))
1111                 rte_exit(EXIT_FAILURE,
1112                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1113                         cur_fwd_eng->fwd_mode_name);
1114
1115         if (all_ports_started() == 0) {
1116                 printf("Not all ports were started\n");
1117                 return;
1118         }
1119         if (test_done == 0) {
1120                 printf("Packet forwarding already started\n");
1121                 return;
1122         }
1123
1124         if (init_fwd_streams() < 0) {
1125                 printf("Fail from init_fwd_streams()\n");
1126                 return;
1127         }
1128
1129         if(dcb_test) {
1130                 for (i = 0; i < nb_fwd_ports; i++) {
1131                         pt_id = fwd_ports_ids[i];
1132                         port = &ports[pt_id];
1133                         if (!port->dcb_flag) {
1134                                 printf("In DCB mode, all forwarding ports must "
1135                                        "be configured in this mode.\n");
1136                                 return;
1137                         }
1138                 }
1139                 if (nb_fwd_lcores == 1) {
1140                         printf("In DCB mode,the nb forwarding cores "
1141                                "should be larger than 1.\n");
1142                         return;
1143                 }
1144         }
1145         test_done = 0;
1146
1147         if(!no_flush_rx)
1148                 flush_fwd_rx_queues();
1149
1150         fwd_config_setup();
1151         pkt_fwd_config_display(&cur_fwd_config);
1152         rxtx_config_display();
1153
1154         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1155                 pt_id = fwd_ports_ids[i];
1156                 port = &ports[pt_id];
1157                 rte_eth_stats_get(pt_id, &port->stats);
1158                 port->tx_dropped = 0;
1159
1160                 map_port_queue_stats_mapping_registers(pt_id, port);
1161         }
1162         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1163                 fwd_streams[sm_id]->rx_packets = 0;
1164                 fwd_streams[sm_id]->tx_packets = 0;
1165                 fwd_streams[sm_id]->fwd_dropped = 0;
1166                 fwd_streams[sm_id]->rx_bad_ip_csum = 0;
1167                 fwd_streams[sm_id]->rx_bad_l4_csum = 0;
1168
1169 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1170                 memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
1171                        sizeof(fwd_streams[sm_id]->rx_burst_stats));
1172                 memset(&fwd_streams[sm_id]->tx_burst_stats, 0,
1173                        sizeof(fwd_streams[sm_id]->tx_burst_stats));
1174 #endif
1175 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1176                 fwd_streams[sm_id]->core_cycles = 0;
1177 #endif
1178         }
1179         if (with_tx_first) {
1180                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1181                 if (port_fwd_begin != NULL) {
1182                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1183                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1184                 }
1185                 while (with_tx_first--) {
1186                         launch_packet_forwarding(
1187                                         run_one_txonly_burst_on_core);
1188                         rte_eal_mp_wait_lcore();
1189                 }
1190                 port_fwd_end = tx_only_engine.port_fwd_end;
1191                 if (port_fwd_end != NULL) {
1192                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1193                                 (*port_fwd_end)(fwd_ports_ids[i]);
1194                 }
1195         }
1196         launch_packet_forwarding(start_pkt_forward_on_core);
1197 }
1198
1199 void
1200 stop_packet_forwarding(void)
1201 {
1202         struct rte_eth_stats stats;
1203         struct rte_port *port;
1204         port_fwd_end_t  port_fwd_end;
1205         int i;
1206         portid_t   pt_id;
1207         streamid_t sm_id;
1208         lcoreid_t  lc_id;
1209         uint64_t total_recv;
1210         uint64_t total_xmit;
1211         uint64_t total_rx_dropped;
1212         uint64_t total_tx_dropped;
1213         uint64_t total_rx_nombuf;
1214         uint64_t tx_dropped;
1215         uint64_t rx_bad_ip_csum;
1216         uint64_t rx_bad_l4_csum;
1217 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1218         uint64_t fwd_cycles;
1219 #endif
1220         static const char *acc_stats_border = "+++++++++++++++";
1221
1222         if (test_done) {
1223                 printf("Packet forwarding not started\n");
1224                 return;
1225         }
1226         printf("Telling cores to stop...");
1227         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1228                 fwd_lcores[lc_id]->stopped = 1;
1229         printf("\nWaiting for lcores to finish...\n");
1230         rte_eal_mp_wait_lcore();
1231         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1232         if (port_fwd_end != NULL) {
1233                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1234                         pt_id = fwd_ports_ids[i];
1235                         (*port_fwd_end)(pt_id);
1236                 }
1237         }
1238 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1239         fwd_cycles = 0;
1240 #endif
1241         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1242                 if (cur_fwd_config.nb_fwd_streams >
1243                     cur_fwd_config.nb_fwd_ports) {
1244                         fwd_stream_stats_display(sm_id);
1245                         ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL;
1246                         ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL;
1247                 } else {
1248                         ports[fwd_streams[sm_id]->tx_port].tx_stream =
1249                                 fwd_streams[sm_id];
1250                         ports[fwd_streams[sm_id]->rx_port].rx_stream =
1251                                 fwd_streams[sm_id];
1252                 }
1253                 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped;
1254                 tx_dropped = (uint64_t) (tx_dropped +
1255                                          fwd_streams[sm_id]->fwd_dropped);
1256                 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped;
1257
1258                 rx_bad_ip_csum =
1259                         ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum;
1260                 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum +
1261                                          fwd_streams[sm_id]->rx_bad_ip_csum);
1262                 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum =
1263                                                         rx_bad_ip_csum;
1264
1265                 rx_bad_l4_csum =
1266                         ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum;
1267                 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum +
1268                                          fwd_streams[sm_id]->rx_bad_l4_csum);
1269                 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum =
1270                                                         rx_bad_l4_csum;
1271
1272 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1273                 fwd_cycles = (uint64_t) (fwd_cycles +
1274                                          fwd_streams[sm_id]->core_cycles);
1275 #endif
1276         }
1277         total_recv = 0;
1278         total_xmit = 0;
1279         total_rx_dropped = 0;
1280         total_tx_dropped = 0;
1281         total_rx_nombuf  = 0;
1282         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1283                 pt_id = fwd_ports_ids[i];
1284
1285                 port = &ports[pt_id];
1286                 rte_eth_stats_get(pt_id, &stats);
1287                 stats.ipackets -= port->stats.ipackets;
1288                 port->stats.ipackets = 0;
1289                 stats.opackets -= port->stats.opackets;
1290                 port->stats.opackets = 0;
1291                 stats.ibytes   -= port->stats.ibytes;
1292                 port->stats.ibytes = 0;
1293                 stats.obytes   -= port->stats.obytes;
1294                 port->stats.obytes = 0;
1295                 stats.imissed  -= port->stats.imissed;
1296                 port->stats.imissed = 0;
1297                 stats.oerrors  -= port->stats.oerrors;
1298                 port->stats.oerrors = 0;
1299                 stats.rx_nombuf -= port->stats.rx_nombuf;
1300                 port->stats.rx_nombuf = 0;
1301
1302                 total_recv += stats.ipackets;
1303                 total_xmit += stats.opackets;
1304                 total_rx_dropped += stats.imissed;
1305                 total_tx_dropped += port->tx_dropped;
1306                 total_rx_nombuf  += stats.rx_nombuf;
1307
1308                 fwd_port_stats_display(pt_id, &stats);
1309         }
1310         printf("\n  %s Accumulated forward statistics for all ports"
1311                "%s\n",
1312                acc_stats_border, acc_stats_border);
1313         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1314                "%-"PRIu64"\n"
1315                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1316                "%-"PRIu64"\n",
1317                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1318                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1319         if (total_rx_nombuf > 0)
1320                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1321         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1322                "%s\n",
1323                acc_stats_border, acc_stats_border);
1324 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1325         if (total_recv > 0)
1326                 printf("\n  CPU cycles/packet=%u (total cycles="
1327                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1328                        (unsigned int)(fwd_cycles / total_recv),
1329                        fwd_cycles, total_recv);
1330 #endif
1331         printf("\nDone.\n");
1332         test_done = 1;
1333 }
1334
1335 void
1336 dev_set_link_up(portid_t pid)
1337 {
1338         if (rte_eth_dev_set_link_up((uint8_t)pid) < 0)
1339                 printf("\nSet link up fail.\n");
1340 }
1341
1342 void
1343 dev_set_link_down(portid_t pid)
1344 {
1345         if (rte_eth_dev_set_link_down((uint8_t)pid) < 0)
1346                 printf("\nSet link down fail.\n");
1347 }
1348
1349 static int
1350 all_ports_started(void)
1351 {
1352         portid_t pi;
1353         struct rte_port *port;
1354
1355         RTE_ETH_FOREACH_DEV(pi) {
1356                 port = &ports[pi];
1357                 /* Check if there is a port which is not started */
1358                 if ((port->port_status != RTE_PORT_STARTED) &&
1359                         (port->slave_flag == 0))
1360                         return 0;
1361         }
1362
1363         /* No port is not started */
1364         return 1;
1365 }
1366
1367 int
1368 all_ports_stopped(void)
1369 {
1370         portid_t pi;
1371         struct rte_port *port;
1372
1373         RTE_ETH_FOREACH_DEV(pi) {
1374                 port = &ports[pi];
1375                 if ((port->port_status != RTE_PORT_STOPPED) &&
1376                         (port->slave_flag == 0))
1377                         return 0;
1378         }
1379
1380         return 1;
1381 }
1382
1383 int
1384 port_is_started(portid_t port_id)
1385 {
1386         if (port_id_is_invalid(port_id, ENABLED_WARN))
1387                 return 0;
1388
1389         if (ports[port_id].port_status != RTE_PORT_STARTED)
1390                 return 0;
1391
1392         return 1;
1393 }
1394
1395 static int
1396 port_is_closed(portid_t port_id)
1397 {
1398         if (port_id_is_invalid(port_id, ENABLED_WARN))
1399                 return 0;
1400
1401         if (ports[port_id].port_status != RTE_PORT_CLOSED)
1402                 return 0;
1403
1404         return 1;
1405 }
1406
1407 int
1408 start_port(portid_t pid)
1409 {
1410         int diag, need_check_link_status = -1;
1411         portid_t pi;
1412         queueid_t qi;
1413         struct rte_port *port;
1414         struct ether_addr mac_addr;
1415         enum rte_eth_event_type event_type;
1416
1417         if (port_id_is_invalid(pid, ENABLED_WARN))
1418                 return 0;
1419
1420         if(dcb_config)
1421                 dcb_test = 1;
1422         RTE_ETH_FOREACH_DEV(pi) {
1423                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1424                         continue;
1425
1426                 need_check_link_status = 0;
1427                 port = &ports[pi];
1428                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1429                                                  RTE_PORT_HANDLING) == 0) {
1430                         printf("Port %d is now not stopped\n", pi);
1431                         continue;
1432                 }
1433
1434                 if (port->need_reconfig > 0) {
1435                         port->need_reconfig = 0;
1436
1437                         if (flow_isolate_all) {
1438                                 int ret = port_flow_isolate(pi, 1);
1439                                 if (ret) {
1440                                         printf("Failed to apply isolated"
1441                                                " mode on port %d\n", pi);
1442                                         return -1;
1443                                 }
1444                         }
1445
1446                         printf("Configuring Port %d (socket %u)\n", pi,
1447                                         port->socket_id);
1448                         /* configure port */
1449                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1450                                                 &(port->dev_conf));
1451                         if (diag != 0) {
1452                                 if (rte_atomic16_cmpset(&(port->port_status),
1453                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1454                                         printf("Port %d can not be set back "
1455                                                         "to stopped\n", pi);
1456                                 printf("Fail to configure port %d\n", pi);
1457                                 /* try to reconfigure port next time */
1458                                 port->need_reconfig = 1;
1459                                 return -1;
1460                         }
1461                 }
1462                 if (port->need_reconfig_queues > 0) {
1463                         port->need_reconfig_queues = 0;
1464                         /* setup tx queues */
1465                         for (qi = 0; qi < nb_txq; qi++) {
1466                                 if ((numa_support) &&
1467                                         (txring_numa[pi] != NUMA_NO_CONFIG))
1468                                         diag = rte_eth_tx_queue_setup(pi, qi,
1469                                                 nb_txd,txring_numa[pi],
1470                                                 &(port->tx_conf));
1471                                 else
1472                                         diag = rte_eth_tx_queue_setup(pi, qi,
1473                                                 nb_txd,port->socket_id,
1474                                                 &(port->tx_conf));
1475
1476                                 if (diag == 0)
1477                                         continue;
1478
1479                                 /* Fail to setup tx queue, return */
1480                                 if (rte_atomic16_cmpset(&(port->port_status),
1481                                                         RTE_PORT_HANDLING,
1482                                                         RTE_PORT_STOPPED) == 0)
1483                                         printf("Port %d can not be set back "
1484                                                         "to stopped\n", pi);
1485                                 printf("Fail to configure port %d tx queues\n", pi);
1486                                 /* try to reconfigure queues next time */
1487                                 port->need_reconfig_queues = 1;
1488                                 return -1;
1489                         }
1490                         /* setup rx queues */
1491                         for (qi = 0; qi < nb_rxq; qi++) {
1492                                 if ((numa_support) &&
1493                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
1494                                         struct rte_mempool * mp =
1495                                                 mbuf_pool_find(rxring_numa[pi]);
1496                                         if (mp == NULL) {
1497                                                 printf("Failed to setup RX queue:"
1498                                                         "No mempool allocation"
1499                                                         " on the socket %d\n",
1500                                                         rxring_numa[pi]);
1501                                                 return -1;
1502                                         }
1503
1504                                         diag = rte_eth_rx_queue_setup(pi, qi,
1505                                              nb_rxd,rxring_numa[pi],
1506                                              &(port->rx_conf),mp);
1507                                 } else {
1508                                         struct rte_mempool *mp =
1509                                                 mbuf_pool_find(port->socket_id);
1510                                         if (mp == NULL) {
1511                                                 printf("Failed to setup RX queue:"
1512                                                         "No mempool allocation"
1513                                                         " on the socket %d\n",
1514                                                         port->socket_id);
1515                                                 return -1;
1516                                         }
1517                                         diag = rte_eth_rx_queue_setup(pi, qi,
1518                                              nb_rxd,port->socket_id,
1519                                              &(port->rx_conf), mp);
1520                                 }
1521                                 if (diag == 0)
1522                                         continue;
1523
1524                                 /* Fail to setup rx queue, return */
1525                                 if (rte_atomic16_cmpset(&(port->port_status),
1526                                                         RTE_PORT_HANDLING,
1527                                                         RTE_PORT_STOPPED) == 0)
1528                                         printf("Port %d can not be set back "
1529                                                         "to stopped\n", pi);
1530                                 printf("Fail to configure port %d rx queues\n", pi);
1531                                 /* try to reconfigure queues next time */
1532                                 port->need_reconfig_queues = 1;
1533                                 return -1;
1534                         }
1535                 }
1536
1537                 for (event_type = RTE_ETH_EVENT_UNKNOWN;
1538                      event_type < RTE_ETH_EVENT_MAX;
1539                      event_type++) {
1540                         diag = rte_eth_dev_callback_register(pi,
1541                                                         event_type,
1542                                                         eth_event_callback,
1543                                                         NULL);
1544                         if (diag) {
1545                                 printf("Failed to setup even callback for event %d\n",
1546                                         event_type);
1547                                 return -1;
1548                         }
1549                 }
1550
1551                 /* start port */
1552                 if (rte_eth_dev_start(pi) < 0) {
1553                         printf("Fail to start port %d\n", pi);
1554
1555                         /* Fail to setup rx queue, return */
1556                         if (rte_atomic16_cmpset(&(port->port_status),
1557                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1558                                 printf("Port %d can not be set back to "
1559                                                         "stopped\n", pi);
1560                         continue;
1561                 }
1562
1563                 if (rte_atomic16_cmpset(&(port->port_status),
1564                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
1565                         printf("Port %d can not be set into started\n", pi);
1566
1567                 rte_eth_macaddr_get(pi, &mac_addr);
1568                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
1569                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
1570                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
1571                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
1572
1573                 /* at least one port started, need checking link status */
1574                 need_check_link_status = 1;
1575         }
1576
1577         if (need_check_link_status == 1 && !no_link_check)
1578                 check_all_ports_link_status(RTE_PORT_ALL);
1579         else if (need_check_link_status == 0)
1580                 printf("Please stop the ports first\n");
1581
1582         printf("Done\n");
1583         return 0;
1584 }
1585
1586 void
1587 stop_port(portid_t pid)
1588 {
1589         portid_t pi;
1590         struct rte_port *port;
1591         int need_check_link_status = 0;
1592
1593         if (dcb_test) {
1594                 dcb_test = 0;
1595                 dcb_config = 0;
1596         }
1597
1598         if (port_id_is_invalid(pid, ENABLED_WARN))
1599                 return;
1600
1601         printf("Stopping ports...\n");
1602
1603         RTE_ETH_FOREACH_DEV(pi) {
1604                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1605                         continue;
1606
1607                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1608                         printf("Please remove port %d from forwarding configuration.\n", pi);
1609                         continue;
1610                 }
1611
1612                 if (port_is_bonding_slave(pi)) {
1613                         printf("Please remove port %d from bonded device.\n", pi);
1614                         continue;
1615                 }
1616
1617                 port = &ports[pi];
1618                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
1619                                                 RTE_PORT_HANDLING) == 0)
1620                         continue;
1621
1622                 rte_eth_dev_stop(pi);
1623
1624                 if (rte_atomic16_cmpset(&(port->port_status),
1625                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1626                         printf("Port %d can not be set into stopped\n", pi);
1627                 need_check_link_status = 1;
1628         }
1629         if (need_check_link_status && !no_link_check)
1630                 check_all_ports_link_status(RTE_PORT_ALL);
1631
1632         printf("Done\n");
1633 }
1634
1635 void
1636 close_port(portid_t pid)
1637 {
1638         portid_t pi;
1639         struct rte_port *port;
1640
1641         if (port_id_is_invalid(pid, ENABLED_WARN))
1642                 return;
1643
1644         printf("Closing ports...\n");
1645
1646         RTE_ETH_FOREACH_DEV(pi) {
1647                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1648                         continue;
1649
1650                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1651                         printf("Please remove port %d from forwarding configuration.\n", pi);
1652                         continue;
1653                 }
1654
1655                 if (port_is_bonding_slave(pi)) {
1656                         printf("Please remove port %d from bonded device.\n", pi);
1657                         continue;
1658                 }
1659
1660                 port = &ports[pi];
1661                 if (rte_atomic16_cmpset(&(port->port_status),
1662                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
1663                         printf("Port %d is already closed\n", pi);
1664                         continue;
1665                 }
1666
1667                 if (rte_atomic16_cmpset(&(port->port_status),
1668                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
1669                         printf("Port %d is now not stopped\n", pi);
1670                         continue;
1671                 }
1672
1673                 if (port->flow_list)
1674                         port_flow_flush(pi);
1675                 rte_eth_dev_close(pi);
1676
1677                 if (rte_atomic16_cmpset(&(port->port_status),
1678                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
1679                         printf("Port %d cannot be set to closed\n", pi);
1680         }
1681
1682         printf("Done\n");
1683 }
1684
1685 void
1686 attach_port(char *identifier)
1687 {
1688         portid_t pi = 0;
1689         unsigned int socket_id;
1690
1691         printf("Attaching a new port...\n");
1692
1693         if (identifier == NULL) {
1694                 printf("Invalid parameters are specified\n");
1695                 return;
1696         }
1697
1698         if (rte_eth_dev_attach(identifier, &pi))
1699                 return;
1700
1701         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
1702         /* if socket_id is invalid, set to 0 */
1703         if (check_socket_id(socket_id) < 0)
1704                 socket_id = 0;
1705         reconfig(pi, socket_id);
1706         rte_eth_promiscuous_enable(pi);
1707
1708         nb_ports = rte_eth_dev_count();
1709
1710         ports[pi].port_status = RTE_PORT_STOPPED;
1711
1712         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
1713         printf("Done\n");
1714 }
1715
1716 void
1717 detach_port(uint8_t port_id)
1718 {
1719         char name[RTE_ETH_NAME_MAX_LEN];
1720
1721         printf("Detaching a port...\n");
1722
1723         if (!port_is_closed(port_id)) {
1724                 printf("Please close port first\n");
1725                 return;
1726         }
1727
1728         if (ports[port_id].flow_list)
1729                 port_flow_flush(port_id);
1730
1731         if (rte_eth_dev_detach(port_id, name))
1732                 return;
1733
1734         nb_ports = rte_eth_dev_count();
1735
1736         printf("Port '%s' is detached. Now total ports is %d\n",
1737                         name, nb_ports);
1738         printf("Done\n");
1739         return;
1740 }
1741
1742 void
1743 pmd_test_exit(void)
1744 {
1745         portid_t pt_id;
1746
1747         if (test_done == 0)
1748                 stop_packet_forwarding();
1749
1750         if (ports != NULL) {
1751                 no_link_check = 1;
1752                 RTE_ETH_FOREACH_DEV(pt_id) {
1753                         printf("\nShutting down port %d...\n", pt_id);
1754                         fflush(stdout);
1755                         stop_port(pt_id);
1756                         close_port(pt_id);
1757                 }
1758         }
1759         printf("\nBye...\n");
1760 }
1761
1762 typedef void (*cmd_func_t)(void);
1763 struct pmd_test_command {
1764         const char *cmd_name;
1765         cmd_func_t cmd_func;
1766 };
1767
1768 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
1769
1770 /* Check the link status of all ports in up to 9s, and print them finally */
1771 static void
1772 check_all_ports_link_status(uint32_t port_mask)
1773 {
1774 #define CHECK_INTERVAL 100 /* 100ms */
1775 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
1776         uint8_t portid, count, all_ports_up, print_flag = 0;
1777         struct rte_eth_link link;
1778
1779         printf("Checking link statuses...\n");
1780         fflush(stdout);
1781         for (count = 0; count <= MAX_CHECK_TIME; count++) {
1782                 all_ports_up = 1;
1783                 RTE_ETH_FOREACH_DEV(portid) {
1784                         if ((port_mask & (1 << portid)) == 0)
1785                                 continue;
1786                         memset(&link, 0, sizeof(link));
1787                         rte_eth_link_get_nowait(portid, &link);
1788                         /* print link status if flag set */
1789                         if (print_flag == 1) {
1790                                 if (link.link_status)
1791                                         printf("Port %d Link Up - speed %u "
1792                                                 "Mbps - %s\n", (uint8_t)portid,
1793                                                 (unsigned)link.link_speed,
1794                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
1795                                         ("full-duplex") : ("half-duplex\n"));
1796                                 else
1797                                         printf("Port %d Link Down\n",
1798                                                 (uint8_t)portid);
1799                                 continue;
1800                         }
1801                         /* clear all_ports_up flag if any link down */
1802                         if (link.link_status == ETH_LINK_DOWN) {
1803                                 all_ports_up = 0;
1804                                 break;
1805                         }
1806                 }
1807                 /* after finally printing all link status, get out */
1808                 if (print_flag == 1)
1809                         break;
1810
1811                 if (all_ports_up == 0) {
1812                         fflush(stdout);
1813                         rte_delay_ms(CHECK_INTERVAL);
1814                 }
1815
1816                 /* set the print_flag if all ports up or timeout */
1817                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
1818                         print_flag = 1;
1819                 }
1820
1821                 if (lsc_interrupt)
1822                         break;
1823         }
1824 }
1825
1826 static void
1827 rmv_event_callback(void *arg)
1828 {
1829         struct rte_eth_dev *dev;
1830         uint8_t port_id = (intptr_t)arg;
1831
1832         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1833         dev = &rte_eth_devices[port_id];
1834
1835         stop_port(port_id);
1836         close_port(port_id);
1837         printf("removing device %s\n", dev->device->name);
1838         rte_eal_dev_detach(dev->device);
1839         dev->state = RTE_ETH_DEV_UNUSED;
1840 }
1841
1842 /* This function is used by the interrupt thread */
1843 static int
1844 eth_event_callback(uint8_t port_id, enum rte_eth_event_type type, void *param,
1845                   void *ret_param)
1846 {
1847         static const char * const event_desc[] = {
1848                 [RTE_ETH_EVENT_UNKNOWN] = "Unknown",
1849                 [RTE_ETH_EVENT_INTR_LSC] = "LSC",
1850                 [RTE_ETH_EVENT_QUEUE_STATE] = "Queue state",
1851                 [RTE_ETH_EVENT_INTR_RESET] = "Interrupt reset",
1852                 [RTE_ETH_EVENT_VF_MBOX] = "VF Mbox",
1853                 [RTE_ETH_EVENT_MACSEC] = "MACsec",
1854                 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
1855                 [RTE_ETH_EVENT_MAX] = NULL,
1856         };
1857
1858         RTE_SET_USED(param);
1859         RTE_SET_USED(ret_param);
1860
1861         if (type >= RTE_ETH_EVENT_MAX) {
1862                 fprintf(stderr, "\nPort %" PRIu8 ": %s called upon invalid event %d\n",
1863                         port_id, __func__, type);
1864                 fflush(stderr);
1865         } else if (event_print_mask & (UINT32_C(1) << type)) {
1866                 printf("\nPort %" PRIu8 ": %s event\n", port_id,
1867                         event_desc[type]);
1868                 fflush(stdout);
1869         }
1870
1871         switch (type) {
1872         case RTE_ETH_EVENT_INTR_RMV:
1873                 if (rte_eal_alarm_set(100000,
1874                                 rmv_event_callback, (void *)(intptr_t)port_id))
1875                         fprintf(stderr, "Could not set up deferred device removal\n");
1876                 break;
1877         default:
1878                 break;
1879         }
1880         return 0;
1881 }
1882
1883 static int
1884 set_tx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port)
1885 {
1886         uint16_t i;
1887         int diag;
1888         uint8_t mapping_found = 0;
1889
1890         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
1891                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
1892                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
1893                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
1894                                         tx_queue_stats_mappings[i].queue_id,
1895                                         tx_queue_stats_mappings[i].stats_counter_id);
1896                         if (diag != 0)
1897                                 return diag;
1898                         mapping_found = 1;
1899                 }
1900         }
1901         if (mapping_found)
1902                 port->tx_queue_stats_mapping_enabled = 1;
1903         return 0;
1904 }
1905
1906 static int
1907 set_rx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port)
1908 {
1909         uint16_t i;
1910         int diag;
1911         uint8_t mapping_found = 0;
1912
1913         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
1914                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
1915                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
1916                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
1917                                         rx_queue_stats_mappings[i].queue_id,
1918                                         rx_queue_stats_mappings[i].stats_counter_id);
1919                         if (diag != 0)
1920                                 return diag;
1921                         mapping_found = 1;
1922                 }
1923         }
1924         if (mapping_found)
1925                 port->rx_queue_stats_mapping_enabled = 1;
1926         return 0;
1927 }
1928
1929 static void
1930 map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port)
1931 {
1932         int diag = 0;
1933
1934         diag = set_tx_queue_stats_mapping_registers(pi, port);
1935         if (diag != 0) {
1936                 if (diag == -ENOTSUP) {
1937                         port->tx_queue_stats_mapping_enabled = 0;
1938                         printf("TX queue stats mapping not supported port id=%d\n", pi);
1939                 }
1940                 else
1941                         rte_exit(EXIT_FAILURE,
1942                                         "set_tx_queue_stats_mapping_registers "
1943                                         "failed for port id=%d diag=%d\n",
1944                                         pi, diag);
1945         }
1946
1947         diag = set_rx_queue_stats_mapping_registers(pi, port);
1948         if (diag != 0) {
1949                 if (diag == -ENOTSUP) {
1950                         port->rx_queue_stats_mapping_enabled = 0;
1951                         printf("RX queue stats mapping not supported port id=%d\n", pi);
1952                 }
1953                 else
1954                         rte_exit(EXIT_FAILURE,
1955                                         "set_rx_queue_stats_mapping_registers "
1956                                         "failed for port id=%d diag=%d\n",
1957                                         pi, diag);
1958         }
1959 }
1960
1961 static void
1962 rxtx_port_config(struct rte_port *port)
1963 {
1964         port->rx_conf = port->dev_info.default_rxconf;
1965         port->tx_conf = port->dev_info.default_txconf;
1966
1967         /* Check if any RX/TX parameters have been passed */
1968         if (rx_pthresh != RTE_PMD_PARAM_UNSET)
1969                 port->rx_conf.rx_thresh.pthresh = rx_pthresh;
1970
1971         if (rx_hthresh != RTE_PMD_PARAM_UNSET)
1972                 port->rx_conf.rx_thresh.hthresh = rx_hthresh;
1973
1974         if (rx_wthresh != RTE_PMD_PARAM_UNSET)
1975                 port->rx_conf.rx_thresh.wthresh = rx_wthresh;
1976
1977         if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
1978                 port->rx_conf.rx_free_thresh = rx_free_thresh;
1979
1980         if (rx_drop_en != RTE_PMD_PARAM_UNSET)
1981                 port->rx_conf.rx_drop_en = rx_drop_en;
1982
1983         if (tx_pthresh != RTE_PMD_PARAM_UNSET)
1984                 port->tx_conf.tx_thresh.pthresh = tx_pthresh;
1985
1986         if (tx_hthresh != RTE_PMD_PARAM_UNSET)
1987                 port->tx_conf.tx_thresh.hthresh = tx_hthresh;
1988
1989         if (tx_wthresh != RTE_PMD_PARAM_UNSET)
1990                 port->tx_conf.tx_thresh.wthresh = tx_wthresh;
1991
1992         if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
1993                 port->tx_conf.tx_rs_thresh = tx_rs_thresh;
1994
1995         if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
1996                 port->tx_conf.tx_free_thresh = tx_free_thresh;
1997
1998         if (txq_flags != RTE_PMD_PARAM_UNSET)
1999                 port->tx_conf.txq_flags = txq_flags;
2000 }
2001
2002 void
2003 init_port_config(void)
2004 {
2005         portid_t pid;
2006         struct rte_port *port;
2007
2008         RTE_ETH_FOREACH_DEV(pid) {
2009                 port = &ports[pid];
2010                 port->dev_conf.rxmode = rx_mode;
2011                 port->dev_conf.fdir_conf = fdir_conf;
2012                 if (nb_rxq > 1) {
2013                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2014                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = rss_hf;
2015                 } else {
2016                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2017                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2018                 }
2019
2020                 if (port->dcb_flag == 0) {
2021                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2022                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2023                         else
2024                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2025                 }
2026
2027                 rxtx_port_config(port);
2028
2029                 rte_eth_macaddr_get(pid, &port->eth_addr);
2030
2031                 map_port_queue_stats_mapping_registers(pid, port);
2032 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2033                 rte_pmd_ixgbe_bypass_init(pid);
2034 #endif
2035
2036                 if (lsc_interrupt &&
2037                     (rte_eth_devices[pid].data->dev_flags &
2038                      RTE_ETH_DEV_INTR_LSC))
2039                         port->dev_conf.intr_conf.lsc = 1;
2040                 if (rmv_interrupt &&
2041                     (rte_eth_devices[pid].data->dev_flags &
2042                      RTE_ETH_DEV_INTR_RMV))
2043                         port->dev_conf.intr_conf.rmv = 1;
2044         }
2045 }
2046
2047 void set_port_slave_flag(portid_t slave_pid)
2048 {
2049         struct rte_port *port;
2050
2051         port = &ports[slave_pid];
2052         port->slave_flag = 1;
2053 }
2054
2055 void clear_port_slave_flag(portid_t slave_pid)
2056 {
2057         struct rte_port *port;
2058
2059         port = &ports[slave_pid];
2060         port->slave_flag = 0;
2061 }
2062
2063 uint8_t port_is_bonding_slave(portid_t slave_pid)
2064 {
2065         struct rte_port *port;
2066
2067         port = &ports[slave_pid];
2068         return port->slave_flag;
2069 }
2070
2071 const uint16_t vlan_tags[] = {
2072                 0,  1,  2,  3,  4,  5,  6,  7,
2073                 8,  9, 10, 11,  12, 13, 14, 15,
2074                 16, 17, 18, 19, 20, 21, 22, 23,
2075                 24, 25, 26, 27, 28, 29, 30, 31
2076 };
2077
2078 static  int
2079 get_eth_dcb_conf(struct rte_eth_conf *eth_conf,
2080                  enum dcb_mode_enable dcb_mode,
2081                  enum rte_eth_nb_tcs num_tcs,
2082                  uint8_t pfc_en)
2083 {
2084         uint8_t i;
2085
2086         /*
2087          * Builds up the correct configuration for dcb+vt based on the vlan tags array
2088          * given above, and the number of traffic classes available for use.
2089          */
2090         if (dcb_mode == DCB_VT_ENABLED) {
2091                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
2092                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
2093                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2094                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
2095
2096                 /* VMDQ+DCB RX and TX configurations */
2097                 vmdq_rx_conf->enable_default_pool = 0;
2098                 vmdq_rx_conf->default_pool = 0;
2099                 vmdq_rx_conf->nb_queue_pools =
2100                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2101                 vmdq_tx_conf->nb_queue_pools =
2102                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2103
2104                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
2105                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
2106                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
2107                         vmdq_rx_conf->pool_map[i].pools =
2108                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
2109                 }
2110                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2111                         vmdq_rx_conf->dcb_tc[i] = i;
2112                         vmdq_tx_conf->dcb_tc[i] = i;
2113                 }
2114
2115                 /* set DCB mode of RX and TX of multiple queues */
2116                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
2117                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
2118         } else {
2119                 struct rte_eth_dcb_rx_conf *rx_conf =
2120                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
2121                 struct rte_eth_dcb_tx_conf *tx_conf =
2122                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
2123
2124                 rx_conf->nb_tcs = num_tcs;
2125                 tx_conf->nb_tcs = num_tcs;
2126
2127                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2128                         rx_conf->dcb_tc[i] = i % num_tcs;
2129                         tx_conf->dcb_tc[i] = i % num_tcs;
2130                 }
2131                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
2132                 eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf;
2133                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
2134         }
2135
2136         if (pfc_en)
2137                 eth_conf->dcb_capability_en =
2138                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
2139         else
2140                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
2141
2142         return 0;
2143 }
2144
2145 int
2146 init_port_dcb_config(portid_t pid,
2147                      enum dcb_mode_enable dcb_mode,
2148                      enum rte_eth_nb_tcs num_tcs,
2149                      uint8_t pfc_en)
2150 {
2151         struct rte_eth_conf port_conf;
2152         struct rte_port *rte_port;
2153         int retval;
2154         uint16_t i;
2155
2156         rte_port = &ports[pid];
2157
2158         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
2159         /* Enter DCB configuration status */
2160         dcb_config = 1;
2161
2162         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
2163         retval = get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en);
2164         if (retval < 0)
2165                 return retval;
2166         port_conf.rxmode.hw_vlan_filter = 1;
2167
2168         /**
2169          * Write the configuration into the device.
2170          * Set the numbers of RX & TX queues to 0, so
2171          * the RX & TX queues will not be setup.
2172          */
2173         (void)rte_eth_dev_configure(pid, 0, 0, &port_conf);
2174
2175         rte_eth_dev_info_get(pid, &rte_port->dev_info);
2176
2177         /* If dev_info.vmdq_pool_base is greater than 0,
2178          * the queue id of vmdq pools is started after pf queues.
2179          */
2180         if (dcb_mode == DCB_VT_ENABLED &&
2181             rte_port->dev_info.vmdq_pool_base > 0) {
2182                 printf("VMDQ_DCB multi-queue mode is nonsensical"
2183                         " for port %d.", pid);
2184                 return -1;
2185         }
2186
2187         /* Assume the ports in testpmd have the same dcb capability
2188          * and has the same number of rxq and txq in dcb mode
2189          */
2190         if (dcb_mode == DCB_VT_ENABLED) {
2191                 if (rte_port->dev_info.max_vfs > 0) {
2192                         nb_rxq = rte_port->dev_info.nb_rx_queues;
2193                         nb_txq = rte_port->dev_info.nb_tx_queues;
2194                 } else {
2195                         nb_rxq = rte_port->dev_info.max_rx_queues;
2196                         nb_txq = rte_port->dev_info.max_tx_queues;
2197                 }
2198         } else {
2199                 /*if vt is disabled, use all pf queues */
2200                 if (rte_port->dev_info.vmdq_pool_base == 0) {
2201                         nb_rxq = rte_port->dev_info.max_rx_queues;
2202                         nb_txq = rte_port->dev_info.max_tx_queues;
2203                 } else {
2204                         nb_rxq = (queueid_t)num_tcs;
2205                         nb_txq = (queueid_t)num_tcs;
2206
2207                 }
2208         }
2209         rx_free_thresh = 64;
2210
2211         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
2212
2213         rxtx_port_config(rte_port);
2214         /* VLAN filter */
2215         rte_port->dev_conf.rxmode.hw_vlan_filter = 1;
2216         for (i = 0; i < RTE_DIM(vlan_tags); i++)
2217                 rx_vft_set(pid, vlan_tags[i], 1);
2218
2219         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
2220         map_port_queue_stats_mapping_registers(pid, rte_port);
2221
2222         rte_port->dcb_flag = 1;
2223
2224         return 0;
2225 }
2226
2227 static void
2228 init_port(void)
2229 {
2230         /* Configuration of Ethernet ports. */
2231         ports = rte_zmalloc("testpmd: ports",
2232                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
2233                             RTE_CACHE_LINE_SIZE);
2234         if (ports == NULL) {
2235                 rte_exit(EXIT_FAILURE,
2236                                 "rte_zmalloc(%d struct rte_port) failed\n",
2237                                 RTE_MAX_ETHPORTS);
2238         }
2239 }
2240
2241 static void
2242 force_quit(void)
2243 {
2244         pmd_test_exit();
2245         prompt_exit();
2246 }
2247
2248 static void
2249 print_stats(void)
2250 {
2251         uint8_t i;
2252         const char clr[] = { 27, '[', '2', 'J', '\0' };
2253         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
2254
2255         /* Clear screen and move to top left */
2256         printf("%s%s", clr, top_left);
2257
2258         printf("\nPort statistics ====================================");
2259         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2260                 nic_stats_display(fwd_ports_ids[i]);
2261 }
2262
2263 static void
2264 signal_handler(int signum)
2265 {
2266         if (signum == SIGINT || signum == SIGTERM) {
2267                 printf("\nSignal %d received, preparing to exit...\n",
2268                                 signum);
2269 #ifdef RTE_LIBRTE_PDUMP
2270                 /* uninitialize packet capture framework */
2271                 rte_pdump_uninit();
2272 #endif
2273 #ifdef RTE_LIBRTE_LATENCY_STATS
2274                 rte_latencystats_uninit();
2275 #endif
2276                 force_quit();
2277                 /* exit with the expected status */
2278                 signal(signum, SIG_DFL);
2279                 kill(getpid(), signum);
2280         }
2281 }
2282
2283 int
2284 main(int argc, char** argv)
2285 {
2286         int  diag;
2287         uint8_t port_id;
2288
2289         signal(SIGINT, signal_handler);
2290         signal(SIGTERM, signal_handler);
2291
2292         diag = rte_eal_init(argc, argv);
2293         if (diag < 0)
2294                 rte_panic("Cannot init EAL\n");
2295
2296 #ifdef RTE_LIBRTE_PDUMP
2297         /* initialize packet capture framework */
2298         rte_pdump_init(NULL);
2299 #endif
2300
2301         nb_ports = (portid_t) rte_eth_dev_count();
2302         if (nb_ports == 0)
2303                 RTE_LOG(WARNING, EAL, "No probed ethernet devices\n");
2304
2305         /* allocate port structures, and init them */
2306         init_port();
2307
2308         set_def_fwd_config();
2309         if (nb_lcores == 0)
2310                 rte_panic("Empty set of forwarding logical cores - check the "
2311                           "core mask supplied in the command parameters\n");
2312
2313         /* Bitrate/latency stats disabled by default */
2314 #ifdef RTE_LIBRTE_BITRATE
2315         bitrate_enabled = 0;
2316 #endif
2317 #ifdef RTE_LIBRTE_LATENCY_STATS
2318         latencystats_enabled = 0;
2319 #endif
2320
2321         argc -= diag;
2322         argv += diag;
2323         if (argc > 1)
2324                 launch_args_parse(argc, argv);
2325
2326         if (tx_first && interactive)
2327                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
2328                                 "interactive mode.\n");
2329         if (!nb_rxq && !nb_txq)
2330                 printf("Warning: Either rx or tx queues should be non-zero\n");
2331
2332         if (nb_rxq > 1 && nb_rxq > nb_txq)
2333                 printf("Warning: nb_rxq=%d enables RSS configuration, "
2334                        "but nb_txq=%d will prevent to fully test it.\n",
2335                        nb_rxq, nb_txq);
2336
2337         init_config();
2338         if (start_port(RTE_PORT_ALL) != 0)
2339                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
2340
2341         /* set all ports to promiscuous mode by default */
2342         RTE_ETH_FOREACH_DEV(port_id)
2343                 rte_eth_promiscuous_enable(port_id);
2344
2345         /* Init metrics library */
2346         rte_metrics_init(rte_socket_id());
2347
2348 #ifdef RTE_LIBRTE_LATENCY_STATS
2349         if (latencystats_enabled != 0) {
2350                 int ret = rte_latencystats_init(1, NULL);
2351                 if (ret)
2352                         printf("Warning: latencystats init()"
2353                                 " returned error %d\n", ret);
2354                 printf("Latencystats running on lcore %d\n",
2355                         latencystats_lcore_id);
2356         }
2357 #endif
2358
2359         /* Setup bitrate stats */
2360 #ifdef RTE_LIBRTE_BITRATE
2361         if (bitrate_enabled != 0) {
2362                 bitrate_data = rte_stats_bitrate_create();
2363                 if (bitrate_data == NULL)
2364                         rte_exit(EXIT_FAILURE,
2365                                 "Could not allocate bitrate data.\n");
2366                 rte_stats_bitrate_reg(bitrate_data);
2367         }
2368 #endif
2369
2370 #ifdef RTE_LIBRTE_CMDLINE
2371         if (strlen(cmdline_filename) != 0)
2372                 cmdline_read_from_file(cmdline_filename);
2373
2374         if (interactive == 1) {
2375                 if (auto_start) {
2376                         printf("Start automatic packet forwarding\n");
2377                         start_packet_forwarding(0);
2378                 }
2379                 prompt();
2380                 pmd_test_exit();
2381         } else
2382 #endif
2383         {
2384                 char c;
2385                 int rc;
2386
2387                 printf("No commandline core given, start packet forwarding\n");
2388                 start_packet_forwarding(tx_first);
2389                 if (stats_period != 0) {
2390                         uint64_t prev_time = 0, cur_time, diff_time = 0;
2391                         uint64_t timer_period;
2392
2393                         /* Convert to number of cycles */
2394                         timer_period = stats_period * rte_get_timer_hz();
2395
2396                         while (1) {
2397                                 cur_time = rte_get_timer_cycles();
2398                                 diff_time += cur_time - prev_time;
2399
2400                                 if (diff_time >= timer_period) {
2401                                         print_stats();
2402                                         /* Reset the timer */
2403                                         diff_time = 0;
2404                                 }
2405                                 /* Sleep to avoid unnecessary checks */
2406                                 prev_time = cur_time;
2407                                 sleep(1);
2408                         }
2409                 }
2410
2411                 printf("Press enter to exit\n");
2412                 rc = read(0, &c, 1);
2413                 pmd_test_exit();
2414                 if (rc < 0)
2415                         return 1;
2416         }
2417
2418         return 0;
2419 }