app/test-pmd/testpmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdarg.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <signal.h>
  38 #include <string.h>
  39 #include <time.h>
  40 #include <fcntl.h>
  41 #include <sys/types.h>
  42 #include <errno.h>
  43
  44 #include <sys/queue.h>
  45 #include <sys/stat.h>
  46
  47 #include <stdint.h>
  48 #include <unistd.h>
  49 #include <inttypes.h>
  50
  51 #include <rte_common.h>
  52 #include <rte_errno.h>
  53 #include <rte_byteorder.h>
  54 #include <rte_log.h>
  55 #include <rte_debug.h>
  56 #include <rte_cycles.h>
  57 #include <rte_memory.h>
  58 #include <rte_memcpy.h>
  59 #include <rte_memzone.h>
  60 #include <rte_launch.h>
  61 #include <rte_eal.h>
  62 #include <rte_alarm.h>
  63 #include <rte_per_lcore.h>
  64 #include <rte_lcore.h>
  65 #include <rte_atomic.h>
  66 #include <rte_branch_prediction.h>
  67 #include <rte_mempool.h>
  68 #include <rte_malloc.h>
  69 #include <rte_mbuf.h>
  70 #include <rte_interrupts.h>
  71 #include <rte_pci.h>
  72 #include <rte_ether.h>
  73 #include <rte_ethdev.h>
  74 #include <rte_dev.h>
  75 #include <rte_string_fns.h>
  76 #ifdef RTE_LIBRTE_PMD_XENVIRT
  77 #include <rte_eth_xenvirt.h>
  78 #endif
  79 #ifdef RTE_LIBRTE_PDUMP
  80 #include <rte_pdump.h>
  81 #endif
  82 #include <rte_flow.h>
  83 #include <rte_metrics.h>
  84 #ifdef RTE_LIBRTE_BITRATE
  85 #include <rte_bitrate.h>
  86 #endif
  87 #ifdef RTE_LIBRTE_LATENCY_STATS
  88 #include <rte_latencystats.h>
  89 #endif
  90
  91 #include "testpmd.h"
  92
  93 uint16_t verbose_level = 0; /**< Silent by default. */
  94
  95 /* use master core for command line ? */
  96 uint8_t interactive = 0;
  97 uint8_t auto_start = 0;
  98 char cmdline_filename[PATH_MAX] = {0};
  99
 100 /*
 101  * NUMA support configuration.
 102  * When set, the NUMA support attempts to dispatch the allocation of the
 103  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 104  * probed ports among the CPU sockets 0 and 1.
 105  * Otherwise, all memory is allocated from CPU socket 0.
 106  */
 107 uint8_t numa_support = 1; /**< numa enabled by default */
 108
 109 /*
 110  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 111  * not configured.
 112  */
 113 uint8_t socket_num = UMA_NO_CONFIG;
 114
 115 /*
 116  * Use ANONYMOUS mapped memory (might be not physically continuous) for mbufs.
 117  */
 118 uint8_t mp_anon = 0;
 119
 120 /*
 121  * Record the Ethernet address of peer target ports to which packets are
 122  * forwarded.
 123  * Must be instantiated with the ethernet addresses of peer traffic generator
 124  * ports.
 125  */
 126 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 127 portid_t nb_peer_eth_addrs = 0;
 128
 129 /*
 130  * Probed Target Environment.
 131  */
 132 struct rte_port *ports;        /**< For all probed ethernet ports. */
 133 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 134 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 135 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 136
 137 /*
 138  * Test Forwarding Configuration.
 139  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 140  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 141  */
 142 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 143 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 144 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 145 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 146
 147 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 148 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 149
 150 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 151 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 152
 153 /*
 154  * Forwarding engines.
 155  */
 156 struct fwd_engine * fwd_engines[] = {
 157         &io_fwd_engine,
 158         &mac_fwd_engine,
 159         &mac_swap_engine,
 160         &flow_gen_engine,
 161         &rx_only_engine,
 162         &tx_only_engine,
 163         &csum_fwd_engine,
 164         &icmp_echo_engine,
 165 #ifdef RTE_LIBRTE_IEEE1588
 166         &ieee1588_fwd_engine,
 167 #endif
 168         NULL,
 169 };
 170
 171 struct fwd_config cur_fwd_config;
 172 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 173 uint32_t retry_enabled;
 174 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 175 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 176
 177 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 178 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 179                                       * specified on command-line. */
 180
 181 /*
 182  * Configuration of packet segments used by the "txonly" processing engine.
 183  */
 184 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 185 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 186         TXONLY_DEF_PACKET_LEN,
 187 };
 188 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 189
 190 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 191 /**< Split policy for packets to TX. */
 192
 193 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 194 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 195
 196 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 197 uint8_t dcb_config = 0;
 198
 199 /* Whether the dcb is in testing status */
 200 uint8_t dcb_test = 0;
 201
 202 /*
 203  * Configurable number of RX/TX queues.
 204  */
 205 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 206 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 207
 208 /*
 209  * Configurable number of RX/TX ring descriptors.
 210  */
 211 #define RTE_TEST_RX_DESC_DEFAULT 128
 212 #define RTE_TEST_TX_DESC_DEFAULT 512
 213 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 214 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 215
 216 #define RTE_PMD_PARAM_UNSET -1
 217 /*
 218  * Configurable values of RX and TX ring threshold registers.
 219  */
 220
 221 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 222 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 223 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 224
 225 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 226 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 227 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 228
 229 /*
 230  * Configurable value of RX free threshold.
 231  */
 232 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 233
 234 /*
 235  * Configurable value of RX drop enable.
 236  */
 237 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 238
 239 /*
 240  * Configurable value of TX free threshold.
 241  */
 242 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 243
 244 /*
 245  * Configurable value of TX RS bit threshold.
 246  */
 247 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 248
 249 /*
 250  * Configurable value of TX queue flags.
 251  */
 252 int32_t txq_flags = RTE_PMD_PARAM_UNSET;
 253
 254 /*
 255  * Receive Side Scaling (RSS) configuration.
 256  */
 257 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 258
 259 /*
 260  * Port topology configuration
 261  */
 262 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 263
 264 /*
 265  * Avoids to flush all the RX streams before starts forwarding.
 266  */
 267 uint8_t no_flush_rx = 0; /* flush by default */
 268
 269 /*
 270  * Avoids to check link status when starting/stopping a port.
 271  */
 272 uint8_t no_link_check = 0; /* check by default */
 273
 274 /*
 275  * Enable link status change notification
 276  */
 277 uint8_t lsc_interrupt = 1; /* enabled by default */
 278
 279 /*
 280  * Enable device removal notification.
 281  */
 282 uint8_t rmv_interrupt = 1; /* enabled by default */
 283
 284 /*
 285  * Display or mask ether events
 286  * Default to all events except VF_MBOX
 287  */
 288 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 289                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 290                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 291                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 292                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 293                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 294
 295 /*
 296  * NIC bypass mode configuration options.
 297  */
 298 #ifdef RTE_NIC_BYPASS
 299
 300 /* The NIC bypass watchdog timeout. */
 301 uint32_t bypass_timeout = RTE_BYPASS_TMT_OFF;
 302
 303 #endif
 304
 305 #ifdef RTE_LIBRTE_LATENCY_STATS
 306
 307 /*
 308  * Set when latency stats is enabled in the commandline
 309  */
 310 uint8_t latencystats_enabled;
 311
 312 /*
 313  * Lcore ID to serive latency statistics.
 314  */
 315 lcoreid_t latencystats_lcore_id = -1;
 316
 317 #endif
 318
 319 /*
 320  * Ethernet device configuration.
 321  */
 322 struct rte_eth_rxmode rx_mode = {
 323         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 324         .split_hdr_size = 0,
 325         .header_split   = 0, /**< Header Split disabled. */
 326         .hw_ip_checksum = 0, /**< IP checksum offload disabled. */
 327         .hw_vlan_filter = 1, /**< VLAN filtering enabled. */
 328         .hw_vlan_strip  = 1, /**< VLAN strip enabled. */
 329         .hw_vlan_extend = 0, /**< Extended VLAN disabled. */
 330         .jumbo_frame    = 0, /**< Jumbo Frame Support disabled. */
 331         .hw_strip_crc   = 1, /**< CRC stripping by hardware enabled. */
 332 };
 333
 334 struct rte_fdir_conf fdir_conf = {
 335         .mode = RTE_FDIR_MODE_NONE,
 336         .pballoc = RTE_FDIR_PBALLOC_64K,
 337         .status = RTE_FDIR_REPORT_STATUS,
 338         .mask = {
 339                 .vlan_tci_mask = 0x0,
 340                 .ipv4_mask     = {
 341                         .src_ip = 0xFFFFFFFF,
 342                         .dst_ip = 0xFFFFFFFF,
 343                 },
 344                 .ipv6_mask     = {
 345                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 346                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 347                 },
 348                 .src_port_mask = 0xFFFF,
 349                 .dst_port_mask = 0xFFFF,
 350                 .mac_addr_byte_mask = 0xFF,
 351                 .tunnel_type_mask = 1,
 352                 .tunnel_id_mask = 0xFFFFFFFF,
 353         },
 354         .drop_queue = 127,
 355 };
 356
 357 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 358
 359 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 360 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 361
 362 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 363 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 364
 365 uint16_t nb_tx_queue_stats_mappings = 0;
 366 uint16_t nb_rx_queue_stats_mappings = 0;
 367
 368 unsigned max_socket = 0;
 369
 370 #ifdef RTE_LIBRTE_BITRATE
 371 /* Bitrate statistics */
 372 struct rte_stats_bitrates *bitrate_data;
 373 lcoreid_t bitrate_lcore_id;
 374 uint8_t bitrate_enabled;
 375 #endif
 376
 377 /* Forward function declarations */
 378 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port);
 379 static void check_all_ports_link_status(uint32_t port_mask);
 380 static void eth_event_callback(uint8_t port_id,
 381                                enum rte_eth_event_type type,
 382                                void *param);
 383
 384 /*
 385  * Check if all the ports are started.
 386  * If yes, return positive value. If not, return zero.
 387  */
 388 static int all_ports_started(void);
 389
 390 /*
 391  * Setup default configuration.
 392  */
 393 static void
 394 set_default_fwd_lcores_config(void)
 395 {
 396         unsigned int i;
 397         unsigned int nb_lc;
 398         unsigned int sock_num;
 399
 400         nb_lc = 0;
 401         for (i = 0; i < RTE_MAX_LCORE; i++) {
 402                 sock_num = rte_lcore_to_socket_id(i) + 1;
 403                 if (sock_num > max_socket) {
 404                         if (sock_num > RTE_MAX_NUMA_NODES)
 405                                 rte_exit(EXIT_FAILURE, "Total sockets greater than %u\n", RTE_MAX_NUMA_NODES);
 406                         max_socket = sock_num;
 407                 }
 408                 if (!rte_lcore_is_enabled(i))
 409                         continue;
 410                 if (i == rte_get_master_lcore())
 411                         continue;
 412                 fwd_lcores_cpuids[nb_lc++] = i;
 413         }
 414         nb_lcores = (lcoreid_t) nb_lc;
 415         nb_cfg_lcores = nb_lcores;
 416         nb_fwd_lcores = 1;
 417 }
 418
 419 static void
 420 set_def_peer_eth_addrs(void)
 421 {
 422         portid_t i;
 423
 424         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 425                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 426                 peer_eth_addrs[i].addr_bytes[5] = i;
 427         }
 428 }
 429
 430 static void
 431 set_default_fwd_ports_config(void)
 432 {
 433         portid_t pt_id;
 434
 435         for (pt_id = 0; pt_id < nb_ports; pt_id++)
 436                 fwd_ports_ids[pt_id] = pt_id;
 437
 438         nb_cfg_ports = nb_ports;
 439         nb_fwd_ports = nb_ports;
 440 }
 441
 442 void
 443 set_def_fwd_config(void)
 444 {
 445         set_default_fwd_lcores_config();
 446         set_def_peer_eth_addrs();
 447         set_default_fwd_ports_config();
 448 }
 449
 450 /*
 451  * Configuration initialisation done once at init time.
 452  */
 453 static void
 454 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 455                  unsigned int socket_id)
 456 {
 457         char pool_name[RTE_MEMPOOL_NAMESIZE];
 458         struct rte_mempool *rte_mp = NULL;
 459         uint32_t mb_size;
 460
 461         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 462         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 463
 464         RTE_LOG(INFO, USER1,
 465                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 466                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 467
 468 #ifdef RTE_LIBRTE_PMD_XENVIRT
 469         rte_mp = rte_mempool_gntalloc_create(pool_name, nb_mbuf, mb_size,
 470                 (unsigned) mb_mempool_cache,
 471                 sizeof(struct rte_pktmbuf_pool_private),
 472                 rte_pktmbuf_pool_init, NULL,
 473                 rte_pktmbuf_init, NULL,
 474                 socket_id, 0);
 475 #endif
 476
 477         /* if the former XEN allocation failed fall back to normal allocation */
 478         if (rte_mp == NULL) {
 479                 if (mp_anon != 0) {
 480                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 481                                 mb_size, (unsigned) mb_mempool_cache,
 482                                 sizeof(struct rte_pktmbuf_pool_private),
 483                                 socket_id, 0);
 484                         if (rte_mp == NULL)
 485                                 goto err;
 486
 487                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 488                                 rte_mempool_free(rte_mp);
 489                                 rte_mp = NULL;
 490                                 goto err;
 491                         }
 492                         rte_pktmbuf_pool_init(rte_mp, NULL);
 493                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 494                 } else {
 495                         /* wrapper to rte_mempool_create() */
 496                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 497                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 498                 }
 499         }
 500
 501 err:
 502         if (rte_mp == NULL) {
 503                 rte_exit(EXIT_FAILURE,
 504                         "Creation of mbuf pool for socket %u failed: %s\n",
 505                         socket_id, rte_strerror(rte_errno));
 506         } else if (verbose_level > 0) {
 507                 rte_mempool_dump(stdout, rte_mp);
 508         }
 509 }
 510
 511 /*
 512  * Check given socket id is valid or not with NUMA mode,
 513  * if valid, return 0, else return -1
 514  */
 515 static int
 516 check_socket_id(const unsigned int socket_id)
 517 {
 518         static int warning_once = 0;
 519
 520         if (socket_id >= max_socket) {
 521                 if (!warning_once && numa_support)
 522                         printf("Warning: NUMA should be configured manually by"
 523                                " using --port-numa-config and"
 524                                " --ring-numa-config parameters along with"
 525                                " --numa.\n");
 526                 warning_once = 1;
 527                 return -1;
 528         }
 529         return 0;
 530 }
 531
 532 static void
 533 init_config(void)
 534 {
 535         portid_t pid;
 536         struct rte_port *port;
 537         struct rte_mempool *mbp;
 538         unsigned int nb_mbuf_per_pool;
 539         lcoreid_t  lc_id;
 540         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
 541
 542         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
 543
 544         if (numa_support) {
 545                 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
 546                 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
 547                 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
 548         }
 549
 550         /* Configuration of logical cores. */
 551         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
 552                                 sizeof(struct fwd_lcore *) * nb_lcores,
 553                                 RTE_CACHE_LINE_SIZE);
 554         if (fwd_lcores == NULL) {
 555                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
 556                                                         "failed\n", nb_lcores);
 557         }
 558         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 559                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
 560                                                sizeof(struct fwd_lcore),
 561                                                RTE_CACHE_LINE_SIZE);
 562                 if (fwd_lcores[lc_id] == NULL) {
 563                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
 564                                                                 "failed\n");
 565                 }
 566                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
 567         }
 568
 569         RTE_ETH_FOREACH_DEV(pid) {
 570                 port = &ports[pid];
 571                 rte_eth_dev_info_get(pid, &port->dev_info);
 572
 573                 if (numa_support) {
 574                         if (port_numa[pid] != NUMA_NO_CONFIG)
 575                                 port_per_socket[port_numa[pid]]++;
 576                         else {
 577                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
 578
 579                                 /* if socket_id is invalid, set to 0 */
 580                                 if (check_socket_id(socket_id) < 0)
 581                                         socket_id = 0;
 582                                 port_per_socket[socket_id]++;
 583                         }
 584                 }
 585
 586                 /* set flag to initialize port/queue */
 587                 port->need_reconfig = 1;
 588                 port->need_reconfig_queues = 1;
 589         }
 590
 591         /*
 592          * Create pools of mbuf.
 593          * If NUMA support is disabled, create a single pool of mbuf in
 594          * socket 0 memory by default.
 595          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
 596          *
 597          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
 598          * nb_txd can be configured at run time.
 599          */
 600         if (param_total_num_mbufs)
 601                 nb_mbuf_per_pool = param_total_num_mbufs;
 602         else {
 603                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
 604                         (nb_lcores * mb_mempool_cache) +
 605                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
 606                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
 607         }
 608
 609         if (numa_support) {
 610                 uint8_t i;
 611
 612                 for (i = 0; i < max_socket; i++)
 613                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, i);
 614         } else {
 615                 if (socket_num == UMA_NO_CONFIG)
 616                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0);
 617                 else
 618                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
 619                                                  socket_num);
 620         }
 621
 622         init_port_config();
 623
 624         /*
 625          * Records which Mbuf pool to use by each logical core, if needed.
 626          */
 627         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 628                 mbp = mbuf_pool_find(
 629                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
 630
 631                 if (mbp == NULL)
 632                         mbp = mbuf_pool_find(0);
 633                 fwd_lcores[lc_id]->mbp = mbp;
 634         }
 635
 636         /* Configuration of packet forwarding streams. */
 637         if (init_fwd_streams() < 0)
 638                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 639
 640         fwd_config_setup();
 641 }
 642
 643
 644 void
 645 reconfig(portid_t new_port_id, unsigned socket_id)
 646 {
 647         struct rte_port *port;
 648
 649         /* Reconfiguration of Ethernet ports. */
 650         port = &ports[new_port_id];
 651         rte_eth_dev_info_get(new_port_id, &port->dev_info);
 652
 653         /* set flag to initialize port/queue */
 654         port->need_reconfig = 1;
 655         port->need_reconfig_queues = 1;
 656         port->socket_id = socket_id;
 657
 658         init_port_config();
 659 }
 660
 661
 662 int
 663 init_fwd_streams(void)
 664 {
 665         portid_t pid;
 666         struct rte_port *port;
 667         streamid_t sm_id, nb_fwd_streams_new;
 668         queueid_t q;
 669
 670         /* set socket id according to numa or not */
 671         RTE_ETH_FOREACH_DEV(pid) {
 672                 port = &ports[pid];
 673                 if (nb_rxq > port->dev_info.max_rx_queues) {
 674                         printf("Fail: nb_rxq(%d) is greater than "
 675                                 "max_rx_queues(%d)\n", nb_rxq,
 676                                 port->dev_info.max_rx_queues);
 677                         return -1;
 678                 }
 679                 if (nb_txq > port->dev_info.max_tx_queues) {
 680                         printf("Fail: nb_txq(%d) is greater than "
 681                                 "max_tx_queues(%d)\n", nb_txq,
 682                                 port->dev_info.max_tx_queues);
 683                         return -1;
 684                 }
 685                 if (numa_support) {
 686                         if (port_numa[pid] != NUMA_NO_CONFIG)
 687                                 port->socket_id = port_numa[pid];
 688                         else {
 689                                 port->socket_id = rte_eth_dev_socket_id(pid);
 690
 691                                 /* if socket_id is invalid, set to 0 */
 692                                 if (check_socket_id(port->socket_id) < 0)
 693                                         port->socket_id = 0;
 694                         }
 695                 }
 696                 else {
 697                         if (socket_num == UMA_NO_CONFIG)
 698                                 port->socket_id = 0;
 699                         else
 700                                 port->socket_id = socket_num;
 701                 }
 702         }
 703
 704         q = RTE_MAX(nb_rxq, nb_txq);
 705         if (q == 0) {
 706                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
 707                 return -1;
 708         }
 709         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
 710         if (nb_fwd_streams_new == nb_fwd_streams)
 711                 return 0;
 712         /* clear the old */
 713         if (fwd_streams != NULL) {
 714                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 715                         if (fwd_streams[sm_id] == NULL)
 716                                 continue;
 717                         rte_free(fwd_streams[sm_id]);
 718                         fwd_streams[sm_id] = NULL;
 719                 }
 720                 rte_free(fwd_streams);
 721                 fwd_streams = NULL;
 722         }
 723
 724         /* init new */
 725         nb_fwd_streams = nb_fwd_streams_new;
 726         fwd_streams = rte_zmalloc("testpmd: fwd_streams",
 727                 sizeof(struct fwd_stream *) * nb_fwd_streams, RTE_CACHE_LINE_SIZE);
 728         if (fwd_streams == NULL)
 729                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_stream *)) "
 730                                                 "failed\n", nb_fwd_streams);
 731
 732         for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 733                 fwd_streams[sm_id] = rte_zmalloc("testpmd: struct fwd_stream",
 734                                 sizeof(struct fwd_stream), RTE_CACHE_LINE_SIZE);
 735                 if (fwd_streams[sm_id] == NULL)
 736                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_stream)"
 737                                                                 " failed\n");
 738         }
 739
 740         return 0;
 741 }
 742
 743 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 744 static void
 745 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
 746 {
 747         unsigned int total_burst;
 748         unsigned int nb_burst;
 749         unsigned int burst_stats[3];
 750         uint16_t pktnb_stats[3];
 751         uint16_t nb_pkt;
 752         int burst_percent[3];
 753
 754         /*
 755          * First compute the total number of packet bursts and the
 756          * two highest numbers of bursts of the same number of packets.
 757          */
 758         total_burst = 0;
 759         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
 760         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
 761         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
 762                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
 763                 if (nb_burst == 0)
 764                         continue;
 765                 total_burst += nb_burst;
 766                 if (nb_burst > burst_stats[0]) {
 767                         burst_stats[1] = burst_stats[0];
 768                         pktnb_stats[1] = pktnb_stats[0];
 769                         burst_stats[0] = nb_burst;
 770                         pktnb_stats[0] = nb_pkt;
 771                 }
 772         }
 773         if (total_burst == 0)
 774                 return;
 775         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
 776         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
 777                burst_percent[0], (int) pktnb_stats[0]);
 778         if (burst_stats[0] == total_burst) {
 779                 printf("]\n");
 780                 return;
 781         }
 782         if (burst_stats[0] + burst_stats[1] == total_burst) {
 783                 printf(" + %d%% of %d pkts]\n",
 784                        100 - burst_percent[0], pktnb_stats[1]);
 785                 return;
 786         }
 787         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
 788         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
 789         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
 790                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
 791                 return;
 792         }
 793         printf(" + %d%% of %d pkts + %d%% of others]\n",
 794                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
 795 }
 796 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
 797
 798 static void
 799 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats)
 800 {
 801         struct rte_port *port;
 802         uint8_t i;
 803
 804         static const char *fwd_stats_border = "----------------------";
 805
 806         port = &ports[port_id];
 807         printf("\n  %s Forward statistics for port %-2d %s\n",
 808                fwd_stats_border, port_id, fwd_stats_border);
 809
 810         if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) {
 811                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
 812                        "%-"PRIu64"\n",
 813                        stats->ipackets, stats->imissed,
 814                        (uint64_t) (stats->ipackets + stats->imissed));
 815
 816                 if (cur_fwd_eng == &csum_fwd_engine)
 817                         printf("  Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n",
 818                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 819                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
 820                         printf("  RX-error: %-"PRIu64"\n",  stats->ierrors);
 821                         printf("  RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf);
 822                 }
 823
 824                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
 825                        "%-"PRIu64"\n",
 826                        stats->opackets, port->tx_dropped,
 827                        (uint64_t) (stats->opackets + port->tx_dropped));
 828         }
 829         else {
 830                 printf("  RX-packets:             %14"PRIu64"    RX-dropped:%14"PRIu64"    RX-total:"
 831                        "%14"PRIu64"\n",
 832                        stats->ipackets, stats->imissed,
 833                        (uint64_t) (stats->ipackets + stats->imissed));
 834
 835                 if (cur_fwd_eng == &csum_fwd_engine)
 836                         printf("  Bad-ipcsum:%14"PRIu64"    Bad-l4csum:%14"PRIu64"\n",
 837                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 838                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
 839                         printf("  RX-error:%"PRIu64"\n", stats->ierrors);
 840                         printf("  RX-nombufs:             %14"PRIu64"\n",
 841                                stats->rx_nombuf);
 842                 }
 843
 844                 printf("  TX-packets:             %14"PRIu64"    TX-dropped:%14"PRIu64"    TX-total:"
 845                        "%14"PRIu64"\n",
 846                        stats->opackets, port->tx_dropped,
 847                        (uint64_t) (stats->opackets + port->tx_dropped));
 848         }
 849
 850 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 851         if (port->rx_stream)
 852                 pkt_burst_stats_display("RX",
 853                         &port->rx_stream->rx_burst_stats);
 854         if (port->tx_stream)
 855                 pkt_burst_stats_display("TX",
 856                         &port->tx_stream->tx_burst_stats);
 857 #endif
 858
 859         if (port->rx_queue_stats_mapping_enabled) {
 860                 printf("\n");
 861                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 862                         printf("  Stats reg %2d RX-packets:%14"PRIu64
 863                                "     RX-errors:%14"PRIu64
 864                                "    RX-bytes:%14"PRIu64"\n",
 865                                i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]);
 866                 }
 867                 printf("\n");
 868         }
 869         if (port->tx_queue_stats_mapping_enabled) {
 870                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 871                         printf("  Stats reg %2d TX-packets:%14"PRIu64
 872                                "                                 TX-bytes:%14"PRIu64"\n",
 873                                i, stats->q_opackets[i], stats->q_obytes[i]);
 874                 }
 875         }
 876
 877         printf("  %s--------------------------------%s\n",
 878                fwd_stats_border, fwd_stats_border);
 879 }
 880
 881 static void
 882 fwd_stream_stats_display(streamid_t stream_id)
 883 {
 884         struct fwd_stream *fs;
 885         static const char *fwd_top_stats_border = "-------";
 886
 887         fs = fwd_streams[stream_id];
 888         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
 889             (fs->fwd_dropped == 0))
 890                 return;
 891         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
 892                "TX Port=%2d/Queue=%2d %s\n",
 893                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
 894                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
 895         printf("  RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u",
 896                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
 897
 898         /* if checksum mode */
 899         if (cur_fwd_eng == &csum_fwd_engine) {
 900                printf("  RX- bad IP checksum: %-14u  Rx- bad L4 checksum: "
 901                         "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum);
 902         }
 903
 904 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 905         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
 906         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
 907 #endif
 908 }
 909
 910 static void
 911 flush_fwd_rx_queues(void)
 912 {
 913         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 914         portid_t  rxp;
 915         portid_t port_id;
 916         queueid_t rxq;
 917         uint16_t  nb_rx;
 918         uint16_t  i;
 919         uint8_t   j;
 920         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
 921         uint64_t timer_period;
 922
 923         /* convert to number of cycles */
 924         timer_period = rte_get_timer_hz(); /* 1 second timeout */
 925
 926         for (j = 0; j < 2; j++) {
 927                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
 928                         for (rxq = 0; rxq < nb_rxq; rxq++) {
 929                                 port_id = fwd_ports_ids[rxp];
 930                                 /**
 931                                 * testpmd can stuck in the below do while loop
 932                                 * if rte_eth_rx_burst() always returns nonzero
 933                                 * packets. So timer is added to exit this loop
 934                                 * after 1sec timer expiry.
 935                                 */
 936                                 prev_tsc = rte_rdtsc();
 937                                 do {
 938                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
 939                                                 pkts_burst, MAX_PKT_BURST);
 940                                         for (i = 0; i < nb_rx; i++)
 941                                                 rte_pktmbuf_free(pkts_burst[i]);
 942
 943                                         cur_tsc = rte_rdtsc();
 944                                         diff_tsc = cur_tsc - prev_tsc;
 945                                         timer_tsc += diff_tsc;
 946                                 } while ((nb_rx > 0) &&
 947                                         (timer_tsc < timer_period));
 948                                 timer_tsc = 0;
 949                         }
 950                 }
 951                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
 952         }
 953 }
 954
 955 static void
 956 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
 957 {
 958         struct fwd_stream **fsm;
 959         streamid_t nb_fs;
 960         streamid_t sm_id;
 961 #ifdef RTE_LIBRTE_BITRATE
 962         uint64_t tics_per_1sec;
 963         uint64_t tics_datum;
 964         uint64_t tics_current;
 965         uint8_t idx_port, cnt_ports;
 966
 967         cnt_ports = rte_eth_dev_count();
 968         tics_datum = rte_rdtsc();
 969         tics_per_1sec = rte_get_timer_hz();
 970 #endif
 971         fsm = &fwd_streams[fc->stream_idx];
 972         nb_fs = fc->stream_nb;
 973         do {
 974                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
 975                         (*pkt_fwd)(fsm[sm_id]);
 976 #ifdef RTE_LIBRTE_BITRATE
 977                 if (bitrate_enabled != 0 &&
 978                                 bitrate_lcore_id == rte_lcore_id()) {
 979                         tics_current = rte_rdtsc();
 980                         if (tics_current - tics_datum >= tics_per_1sec) {
 981                                 /* Periodic bitrate calculation */
 982                                 for (idx_port = 0;
 983                                                 idx_port < cnt_ports;
 984                                                 idx_port++)
 985                                         rte_stats_bitrate_calc(bitrate_data,
 986                                                 idx_port);
 987                                 tics_datum = tics_current;
 988                         }
 989                 }
 990 #endif
 991 #ifdef RTE_LIBRTE_LATENCY_STATS
 992                 if (latencystats_enabled != 0 &&
 993                                 latencystats_lcore_id == rte_lcore_id())
 994                         rte_latencystats_update();
 995 #endif
 996
 997         } while (! fc->stopped);
 998 }
 999
1000 static int
1001 start_pkt_forward_on_core(void *fwd_arg)
1002 {
1003         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1004                              cur_fwd_config.fwd_eng->packet_fwd);
1005         return 0;
1006 }
1007
1008 /*
1009  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1010  * Used to start communication flows in network loopback test configurations.
1011  */
1012 static int
1013 run_one_txonly_burst_on_core(void *fwd_arg)
1014 {
1015         struct fwd_lcore *fwd_lc;
1016         struct fwd_lcore tmp_lcore;
1017
1018         fwd_lc = (struct fwd_lcore *) fwd_arg;
1019         tmp_lcore = *fwd_lc;
1020         tmp_lcore.stopped = 1;
1021         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1022         return 0;
1023 }
1024
1025 /*
1026  * Launch packet forwarding:
1027  *     - Setup per-port forwarding context.
1028  *     - launch logical cores with their forwarding configuration.
1029  */
1030 static void
1031 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1032 {
1033         port_fwd_begin_t port_fwd_begin;
1034         unsigned int i;
1035         unsigned int lc_id;
1036         int diag;
1037
1038         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1039         if (port_fwd_begin != NULL) {
1040                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1041                         (*port_fwd_begin)(fwd_ports_ids[i]);
1042         }
1043         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1044                 lc_id = fwd_lcores_cpuids[i];
1045                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1046                         fwd_lcores[i]->stopped = 0;
1047                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1048                                                      fwd_lcores[i], lc_id);
1049                         if (diag != 0)
1050                                 printf("launch lcore %u failed - diag=%d\n",
1051                                        lc_id, diag);
1052                 }
1053         }
1054 }
1055
1056 /*
1057  * Launch packet forwarding configuration.
1058  */
1059 void
1060 start_packet_forwarding(int with_tx_first)
1061 {
1062         port_fwd_begin_t port_fwd_begin;
1063         port_fwd_end_t  port_fwd_end;
1064         struct rte_port *port;
1065         unsigned int i;
1066         portid_t   pt_id;
1067         streamid_t sm_id;
1068
1069         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1070                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1071
1072         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1073                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1074
1075         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1076                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1077                 (!nb_rxq || !nb_txq))
1078                 rte_exit(EXIT_FAILURE,
1079                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1080                         cur_fwd_eng->fwd_mode_name);
1081
1082         if (all_ports_started() == 0) {
1083                 printf("Not all ports were started\n");
1084                 return;
1085         }
1086         if (test_done == 0) {
1087                 printf("Packet forwarding already started\n");
1088                 return;
1089         }
1090
1091         if (init_fwd_streams() < 0) {
1092                 printf("Fail from init_fwd_streams()\n");
1093                 return;
1094         }
1095
1096         if(dcb_test) {
1097                 for (i = 0; i < nb_fwd_ports; i++) {
1098                         pt_id = fwd_ports_ids[i];
1099                         port = &ports[pt_id];
1100                         if (!port->dcb_flag) {
1101                                 printf("In DCB mode, all forwarding ports must "
1102                                        "be configured in this mode.\n");
1103                                 return;
1104                         }
1105                 }
1106                 if (nb_fwd_lcores == 1) {
1107                         printf("In DCB mode,the nb forwarding cores "
1108                                "should be larger than 1.\n");
1109                         return;
1110                 }
1111         }
1112         test_done = 0;
1113
1114         if(!no_flush_rx)
1115                 flush_fwd_rx_queues();
1116
1117         fwd_config_setup();
1118         pkt_fwd_config_display(&cur_fwd_config);
1119         rxtx_config_display();
1120
1121         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1122                 pt_id = fwd_ports_ids[i];
1123                 port = &ports[pt_id];
1124                 rte_eth_stats_get(pt_id, &port->stats);
1125                 port->tx_dropped = 0;
1126
1127                 map_port_queue_stats_mapping_registers(pt_id, port);
1128         }
1129         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1130                 fwd_streams[sm_id]->rx_packets = 0;
1131                 fwd_streams[sm_id]->tx_packets = 0;
1132                 fwd_streams[sm_id]->fwd_dropped = 0;
1133                 fwd_streams[sm_id]->rx_bad_ip_csum = 0;
1134                 fwd_streams[sm_id]->rx_bad_l4_csum = 0;
1135
1136 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1137                 memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
1138                        sizeof(fwd_streams[sm_id]->rx_burst_stats));
1139                 memset(&fwd_streams[sm_id]->tx_burst_stats, 0,
1140                        sizeof(fwd_streams[sm_id]->tx_burst_stats));
1141 #endif
1142 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1143                 fwd_streams[sm_id]->core_cycles = 0;
1144 #endif
1145         }
1146         if (with_tx_first) {
1147                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1148                 if (port_fwd_begin != NULL) {
1149                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1150                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1151                 }
1152                 while (with_tx_first--) {
1153                         launch_packet_forwarding(
1154                                         run_one_txonly_burst_on_core);
1155                         rte_eal_mp_wait_lcore();
1156                 }
1157                 port_fwd_end = tx_only_engine.port_fwd_end;
1158                 if (port_fwd_end != NULL) {
1159                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1160                                 (*port_fwd_end)(fwd_ports_ids[i]);
1161                 }
1162         }
1163         launch_packet_forwarding(start_pkt_forward_on_core);
1164 }
1165
1166 void
1167 stop_packet_forwarding(void)
1168 {
1169         struct rte_eth_stats stats;
1170         struct rte_port *port;
1171         port_fwd_end_t  port_fwd_end;
1172         int i;
1173         portid_t   pt_id;
1174         streamid_t sm_id;
1175         lcoreid_t  lc_id;
1176         uint64_t total_recv;
1177         uint64_t total_xmit;
1178         uint64_t total_rx_dropped;
1179         uint64_t total_tx_dropped;
1180         uint64_t total_rx_nombuf;
1181         uint64_t tx_dropped;
1182         uint64_t rx_bad_ip_csum;
1183         uint64_t rx_bad_l4_csum;
1184 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1185         uint64_t fwd_cycles;
1186 #endif
1187         static const char *acc_stats_border = "+++++++++++++++";
1188
1189         if (test_done) {
1190                 printf("Packet forwarding not started\n");
1191                 return;
1192         }
1193         printf("Telling cores to stop...");
1194         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1195                 fwd_lcores[lc_id]->stopped = 1;
1196         printf("\nWaiting for lcores to finish...\n");
1197         rte_eal_mp_wait_lcore();
1198         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1199         if (port_fwd_end != NULL) {
1200                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1201                         pt_id = fwd_ports_ids[i];
1202                         (*port_fwd_end)(pt_id);
1203                 }
1204         }
1205 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1206         fwd_cycles = 0;
1207 #endif
1208         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1209                 if (cur_fwd_config.nb_fwd_streams >
1210                     cur_fwd_config.nb_fwd_ports) {
1211                         fwd_stream_stats_display(sm_id);
1212                         ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL;
1213                         ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL;
1214                 } else {
1215                         ports[fwd_streams[sm_id]->tx_port].tx_stream =
1216                                 fwd_streams[sm_id];
1217                         ports[fwd_streams[sm_id]->rx_port].rx_stream =
1218                                 fwd_streams[sm_id];
1219                 }
1220                 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped;
1221                 tx_dropped = (uint64_t) (tx_dropped +
1222                                          fwd_streams[sm_id]->fwd_dropped);
1223                 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped;
1224
1225                 rx_bad_ip_csum =
1226                         ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum;
1227                 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum +
1228                                          fwd_streams[sm_id]->rx_bad_ip_csum);
1229                 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum =
1230                                                         rx_bad_ip_csum;
1231
1232                 rx_bad_l4_csum =
1233                         ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum;
1234                 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum +
1235                                          fwd_streams[sm_id]->rx_bad_l4_csum);
1236                 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum =
1237                                                         rx_bad_l4_csum;
1238
1239 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1240                 fwd_cycles = (uint64_t) (fwd_cycles +
1241                                          fwd_streams[sm_id]->core_cycles);
1242 #endif
1243         }
1244         total_recv = 0;
1245         total_xmit = 0;
1246         total_rx_dropped = 0;
1247         total_tx_dropped = 0;
1248         total_rx_nombuf  = 0;
1249         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1250                 pt_id = fwd_ports_ids[i];
1251
1252                 port = &ports[pt_id];
1253                 rte_eth_stats_get(pt_id, &stats);
1254                 stats.ipackets -= port->stats.ipackets;
1255                 port->stats.ipackets = 0;
1256                 stats.opackets -= port->stats.opackets;
1257                 port->stats.opackets = 0;
1258                 stats.ibytes   -= port->stats.ibytes;
1259                 port->stats.ibytes = 0;
1260                 stats.obytes   -= port->stats.obytes;
1261                 port->stats.obytes = 0;
1262                 stats.imissed  -= port->stats.imissed;
1263                 port->stats.imissed = 0;
1264                 stats.oerrors  -= port->stats.oerrors;
1265                 port->stats.oerrors = 0;
1266                 stats.rx_nombuf -= port->stats.rx_nombuf;
1267                 port->stats.rx_nombuf = 0;
1268
1269                 total_recv += stats.ipackets;
1270                 total_xmit += stats.opackets;
1271                 total_rx_dropped += stats.imissed;
1272                 total_tx_dropped += port->tx_dropped;
1273                 total_rx_nombuf  += stats.rx_nombuf;
1274
1275                 fwd_port_stats_display(pt_id, &stats);
1276         }
1277         printf("\n  %s Accumulated forward statistics for all ports"
1278                "%s\n",
1279                acc_stats_border, acc_stats_border);
1280         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1281                "%-"PRIu64"\n"
1282                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1283                "%-"PRIu64"\n",
1284                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1285                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1286         if (total_rx_nombuf > 0)
1287                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1288         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1289                "%s\n",
1290                acc_stats_border, acc_stats_border);
1291 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1292         if (total_recv > 0)
1293                 printf("\n  CPU cycles/packet=%u (total cycles="
1294                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1295                        (unsigned int)(fwd_cycles / total_recv),
1296                        fwd_cycles, total_recv);
1297 #endif
1298         printf("\nDone.\n");
1299         test_done = 1;
1300 }
1301
1302 void
1303 dev_set_link_up(portid_t pid)
1304 {
1305         if (rte_eth_dev_set_link_up((uint8_t)pid) < 0)
1306                 printf("\nSet link up fail.\n");
1307 }
1308
1309 void
1310 dev_set_link_down(portid_t pid)
1311 {
1312         if (rte_eth_dev_set_link_down((uint8_t)pid) < 0)
1313                 printf("\nSet link down fail.\n");
1314 }
1315
1316 static int
1317 all_ports_started(void)
1318 {
1319         portid_t pi;
1320         struct rte_port *port;
1321
1322         RTE_ETH_FOREACH_DEV(pi) {
1323                 port = &ports[pi];
1324                 /* Check if there is a port which is not started */
1325                 if ((port->port_status != RTE_PORT_STARTED) &&
1326                         (port->slave_flag == 0))
1327                         return 0;
1328         }
1329
1330         /* No port is not started */
1331         return 1;
1332 }
1333
1334 int
1335 all_ports_stopped(void)
1336 {
1337         portid_t pi;
1338         struct rte_port *port;
1339
1340         RTE_ETH_FOREACH_DEV(pi) {
1341                 port = &ports[pi];
1342                 if ((port->port_status != RTE_PORT_STOPPED) &&
1343                         (port->slave_flag == 0))
1344                         return 0;
1345         }
1346
1347         return 1;
1348 }
1349
1350 int
1351 port_is_started(portid_t port_id)
1352 {
1353         if (port_id_is_invalid(port_id, ENABLED_WARN))
1354                 return 0;
1355
1356         if (ports[port_id].port_status != RTE_PORT_STARTED)
1357                 return 0;
1358
1359         return 1;
1360 }
1361
1362 static int
1363 port_is_closed(portid_t port_id)
1364 {
1365         if (port_id_is_invalid(port_id, ENABLED_WARN))
1366                 return 0;
1367
1368         if (ports[port_id].port_status != RTE_PORT_CLOSED)
1369                 return 0;
1370
1371         return 1;
1372 }
1373
1374 int
1375 start_port(portid_t pid)
1376 {
1377         int diag, need_check_link_status = -1;
1378         portid_t pi;
1379         queueid_t qi;
1380         struct rte_port *port;
1381         struct ether_addr mac_addr;
1382         enum rte_eth_event_type event_type;
1383
1384         if (port_id_is_invalid(pid, ENABLED_WARN))
1385                 return 0;
1386
1387         if(dcb_config)
1388                 dcb_test = 1;
1389         RTE_ETH_FOREACH_DEV(pi) {
1390                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1391                         continue;
1392
1393                 need_check_link_status = 0;
1394                 port = &ports[pi];
1395                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1396                                                  RTE_PORT_HANDLING) == 0) {
1397                         printf("Port %d is now not stopped\n", pi);
1398                         continue;
1399                 }
1400
1401                 if (port->need_reconfig > 0) {
1402                         port->need_reconfig = 0;
1403
1404                         printf("Configuring Port %d (socket %u)\n", pi,
1405                                         port->socket_id);
1406                         /* configure port */
1407                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1408                                                 &(port->dev_conf));
1409                         if (diag != 0) {
1410                                 if (rte_atomic16_cmpset(&(port->port_status),
1411                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1412                                         printf("Port %d can not be set back "
1413                                                         "to stopped\n", pi);
1414                                 printf("Fail to configure port %d\n", pi);
1415                                 /* try to reconfigure port next time */
1416                                 port->need_reconfig = 1;
1417                                 return -1;
1418                         }
1419                 }
1420                 if (port->need_reconfig_queues > 0) {
1421                         port->need_reconfig_queues = 0;
1422                         /* setup tx queues */
1423                         for (qi = 0; qi < nb_txq; qi++) {
1424                                 if ((numa_support) &&
1425                                         (txring_numa[pi] != NUMA_NO_CONFIG))
1426                                         diag = rte_eth_tx_queue_setup(pi, qi,
1427                                                 nb_txd,txring_numa[pi],
1428                                                 &(port->tx_conf));
1429                                 else
1430                                         diag = rte_eth_tx_queue_setup(pi, qi,
1431                                                 nb_txd,port->socket_id,
1432                                                 &(port->tx_conf));
1433
1434                                 if (diag == 0)
1435                                         continue;
1436
1437                                 /* Fail to setup tx queue, return */
1438                                 if (rte_atomic16_cmpset(&(port->port_status),
1439                                                         RTE_PORT_HANDLING,
1440                                                         RTE_PORT_STOPPED) == 0)
1441                                         printf("Port %d can not be set back "
1442                                                         "to stopped\n", pi);
1443                                 printf("Fail to configure port %d tx queues\n", pi);
1444                                 /* try to reconfigure queues next time */
1445                                 port->need_reconfig_queues = 1;
1446                                 return -1;
1447                         }
1448                         /* setup rx queues */
1449                         for (qi = 0; qi < nb_rxq; qi++) {
1450                                 if ((numa_support) &&
1451                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
1452                                         struct rte_mempool * mp =
1453                                                 mbuf_pool_find(rxring_numa[pi]);
1454                                         if (mp == NULL) {
1455                                                 printf("Failed to setup RX queue:"
1456                                                         "No mempool allocation"
1457                                                         " on the socket %d\n",
1458                                                         rxring_numa[pi]);
1459                                                 return -1;
1460                                         }
1461
1462                                         diag = rte_eth_rx_queue_setup(pi, qi,
1463                                              nb_rxd,rxring_numa[pi],
1464                                              &(port->rx_conf),mp);
1465                                 } else {
1466                                         struct rte_mempool *mp =
1467                                                 mbuf_pool_find(port->socket_id);
1468                                         if (mp == NULL) {
1469                                                 printf("Failed to setup RX queue:"
1470                                                         "No mempool allocation"
1471                                                         " on the socket %d\n",
1472                                                         port->socket_id);
1473                                                 return -1;
1474                                         }
1475                                         diag = rte_eth_rx_queue_setup(pi, qi,
1476                                              nb_rxd,port->socket_id,
1477                                              &(port->rx_conf), mp);
1478                                 }
1479                                 if (diag == 0)
1480                                         continue;
1481
1482                                 /* Fail to setup rx queue, return */
1483                                 if (rte_atomic16_cmpset(&(port->port_status),
1484                                                         RTE_PORT_HANDLING,
1485                                                         RTE_PORT_STOPPED) == 0)
1486                                         printf("Port %d can not be set back "
1487                                                         "to stopped\n", pi);
1488                                 printf("Fail to configure port %d rx queues\n", pi);
1489                                 /* try to reconfigure queues next time */
1490                                 port->need_reconfig_queues = 1;
1491                                 return -1;
1492                         }
1493                 }
1494
1495                 for (event_type = RTE_ETH_EVENT_UNKNOWN;
1496                      event_type < RTE_ETH_EVENT_MAX;
1497                      event_type++) {
1498                         diag = rte_eth_dev_callback_register(pi,
1499                                                         event_type,
1500                                                         eth_event_callback,
1501                                                         NULL);
1502                         if (diag) {
1503                                 printf("Failed to setup even callback for event %d\n",
1504                                         event_type);
1505                                 return -1;
1506                         }
1507                 }
1508
1509                 /* start port */
1510                 if (rte_eth_dev_start(pi) < 0) {
1511                         printf("Fail to start port %d\n", pi);
1512
1513                         /* Fail to setup rx queue, return */
1514                         if (rte_atomic16_cmpset(&(port->port_status),
1515                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1516                                 printf("Port %d can not be set back to "
1517                                                         "stopped\n", pi);
1518                         continue;
1519                 }
1520
1521                 if (rte_atomic16_cmpset(&(port->port_status),
1522                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
1523                         printf("Port %d can not be set into started\n", pi);
1524
1525                 rte_eth_macaddr_get(pi, &mac_addr);
1526                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
1527                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
1528                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
1529                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
1530
1531                 /* at least one port started, need checking link status */
1532                 need_check_link_status = 1;
1533         }
1534
1535         if (need_check_link_status == 1 && !no_link_check)
1536                 check_all_ports_link_status(RTE_PORT_ALL);
1537         else if (need_check_link_status == 0)
1538                 printf("Please stop the ports first\n");
1539
1540         printf("Done\n");
1541         return 0;
1542 }
1543
1544 void
1545 stop_port(portid_t pid)
1546 {
1547         portid_t pi;
1548         struct rte_port *port;
1549         int need_check_link_status = 0;
1550
1551         if (dcb_test) {
1552                 dcb_test = 0;
1553                 dcb_config = 0;
1554         }
1555
1556         if (port_id_is_invalid(pid, ENABLED_WARN))
1557                 return;
1558
1559         printf("Stopping ports...\n");
1560
1561         RTE_ETH_FOREACH_DEV(pi) {
1562                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1563                         continue;
1564
1565                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1566                         printf("Please remove port %d from forwarding configuration.\n", pi);
1567                         continue;
1568                 }
1569
1570                 if (port_is_bonding_slave(pi)) {
1571                         printf("Please remove port %d from bonded device.\n", pi);
1572                         continue;
1573                 }
1574
1575                 port = &ports[pi];
1576                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
1577                                                 RTE_PORT_HANDLING) == 0)
1578                         continue;
1579
1580                 rte_eth_dev_stop(pi);
1581
1582                 if (rte_atomic16_cmpset(&(port->port_status),
1583                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1584                         printf("Port %d can not be set into stopped\n", pi);
1585                 need_check_link_status = 1;
1586         }
1587         if (need_check_link_status && !no_link_check)
1588                 check_all_ports_link_status(RTE_PORT_ALL);
1589
1590         printf("Done\n");
1591 }
1592
1593 void
1594 close_port(portid_t pid)
1595 {
1596         portid_t pi;
1597         struct rte_port *port;
1598
1599         if (port_id_is_invalid(pid, ENABLED_WARN))
1600                 return;
1601
1602         printf("Closing ports...\n");
1603
1604         RTE_ETH_FOREACH_DEV(pi) {
1605                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1606                         continue;
1607
1608                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1609                         printf("Please remove port %d from forwarding configuration.\n", pi);
1610                         continue;
1611                 }
1612
1613                 if (port_is_bonding_slave(pi)) {
1614                         printf("Please remove port %d from bonded device.\n", pi);
1615                         continue;
1616                 }
1617
1618                 port = &ports[pi];
1619                 if (rte_atomic16_cmpset(&(port->port_status),
1620                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
1621                         printf("Port %d is already closed\n", pi);
1622                         continue;
1623                 }
1624
1625                 if (rte_atomic16_cmpset(&(port->port_status),
1626                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
1627                         printf("Port %d is now not stopped\n", pi);
1628                         continue;
1629                 }
1630
1631                 if (port->flow_list)
1632                         port_flow_flush(pi);
1633                 rte_eth_dev_close(pi);
1634
1635                 if (rte_atomic16_cmpset(&(port->port_status),
1636                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
1637                         printf("Port %d cannot be set to closed\n", pi);
1638         }
1639
1640         printf("Done\n");
1641 }
1642
1643 void
1644 attach_port(char *identifier)
1645 {
1646         portid_t pi = 0;
1647         unsigned int socket_id;
1648
1649         printf("Attaching a new port...\n");
1650
1651         if (identifier == NULL) {
1652                 printf("Invalid parameters are specified\n");
1653                 return;
1654         }
1655
1656         if (rte_eth_dev_attach(identifier, &pi))
1657                 return;
1658
1659         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
1660         /* if socket_id is invalid, set to 0 */
1661         if (check_socket_id(socket_id) < 0)
1662                 socket_id = 0;
1663         reconfig(pi, socket_id);
1664         rte_eth_promiscuous_enable(pi);
1665
1666         nb_ports = rte_eth_dev_count();
1667
1668         ports[pi].port_status = RTE_PORT_STOPPED;
1669
1670         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
1671         printf("Done\n");
1672 }
1673
1674 void
1675 detach_port(uint8_t port_id)
1676 {
1677         char name[RTE_ETH_NAME_MAX_LEN];
1678
1679         printf("Detaching a port...\n");
1680
1681         if (!port_is_closed(port_id)) {
1682                 printf("Please close port first\n");
1683                 return;
1684         }
1685
1686         if (ports[port_id].flow_list)
1687                 port_flow_flush(port_id);
1688
1689         if (rte_eth_dev_detach(port_id, name))
1690                 return;
1691
1692         nb_ports = rte_eth_dev_count();
1693
1694         printf("Port '%s' is detached. Now total ports is %d\n",
1695                         name, nb_ports);
1696         printf("Done\n");
1697         return;
1698 }
1699
1700 void
1701 pmd_test_exit(void)
1702 {
1703         portid_t pt_id;
1704
1705         if (test_done == 0)
1706                 stop_packet_forwarding();
1707
1708         if (ports != NULL) {
1709                 no_link_check = 1;
1710                 RTE_ETH_FOREACH_DEV(pt_id) {
1711                         printf("\nShutting down port %d...\n", pt_id);
1712                         fflush(stdout);
1713                         stop_port(pt_id);
1714                         close_port(pt_id);
1715                 }
1716         }
1717         printf("\nBye...\n");
1718 }
1719
1720 typedef void (*cmd_func_t)(void);
1721 struct pmd_test_command {
1722         const char *cmd_name;
1723         cmd_func_t cmd_func;
1724 };
1725
1726 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
1727
1728 /* Check the link status of all ports in up to 9s, and print them finally */
1729 static void
1730 check_all_ports_link_status(uint32_t port_mask)
1731 {
1732 #define CHECK_INTERVAL 100 /* 100ms */
1733 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
1734         uint8_t portid, count, all_ports_up, print_flag = 0;
1735         struct rte_eth_link link;
1736
1737         printf("Checking link statuses...\n");
1738         fflush(stdout);
1739         for (count = 0; count <= MAX_CHECK_TIME; count++) {
1740                 all_ports_up = 1;
1741                 RTE_ETH_FOREACH_DEV(portid) {
1742                         if ((port_mask & (1 << portid)) == 0)
1743                                 continue;
1744                         memset(&link, 0, sizeof(link));
1745                         rte_eth_link_get_nowait(portid, &link);
1746                         /* print link status if flag set */
1747                         if (print_flag == 1) {
1748                                 if (link.link_status)
1749                                         printf("Port %d Link Up - speed %u "
1750                                                 "Mbps - %s\n", (uint8_t)portid,
1751                                                 (unsigned)link.link_speed,
1752                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
1753                                         ("full-duplex") : ("half-duplex\n"));
1754                                 else
1755                                         printf("Port %d Link Down\n",
1756                                                 (uint8_t)portid);
1757                                 continue;
1758                         }
1759                         /* clear all_ports_up flag if any link down */
1760                         if (link.link_status == ETH_LINK_DOWN) {
1761                                 all_ports_up = 0;
1762                                 break;
1763                         }
1764                 }
1765                 /* after finally printing all link status, get out */
1766                 if (print_flag == 1)
1767                         break;
1768
1769                 if (all_ports_up == 0) {
1770                         fflush(stdout);
1771                         rte_delay_ms(CHECK_INTERVAL);
1772                 }
1773
1774                 /* set the print_flag if all ports up or timeout */
1775                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
1776                         print_flag = 1;
1777                 }
1778
1779                 if (lsc_interrupt)
1780                         break;
1781         }
1782 }
1783
1784 static void
1785 rmv_event_callback(void *arg)
1786 {
1787         struct rte_eth_dev *dev;
1788         struct rte_devargs *da;
1789         char name[32] = "";
1790         uint8_t port_id = (intptr_t)arg;
1791
1792         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1793         dev = &rte_eth_devices[port_id];
1794         da = dev->device->devargs;
1795
1796         stop_port(port_id);
1797         close_port(port_id);
1798         if (da->type == RTE_DEVTYPE_VIRTUAL)
1799                 snprintf(name, sizeof(name), "%s", da->virt.drv_name);
1800         else if (da->type == RTE_DEVTYPE_WHITELISTED_PCI)
1801                 rte_pci_device_name(&da->pci.addr, name, sizeof(name));
1802         printf("removing device %s\n", name);
1803         rte_eal_dev_detach(name);
1804         dev->state = RTE_ETH_DEV_UNUSED;
1805 }
1806
1807 /* This function is used by the interrupt thread */
1808 static void
1809 eth_event_callback(uint8_t port_id, enum rte_eth_event_type type, void *param)
1810 {
1811         static const char * const event_desc[] = {
1812                 [RTE_ETH_EVENT_UNKNOWN] = "Unknown",
1813                 [RTE_ETH_EVENT_INTR_LSC] = "LSC",
1814                 [RTE_ETH_EVENT_QUEUE_STATE] = "Queue state",
1815                 [RTE_ETH_EVENT_INTR_RESET] = "Interrupt reset",
1816                 [RTE_ETH_EVENT_VF_MBOX] = "VF Mbox",
1817                 [RTE_ETH_EVENT_MACSEC] = "MACsec",
1818                 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
1819                 [RTE_ETH_EVENT_MAX] = NULL,
1820         };
1821
1822         RTE_SET_USED(param);
1823
1824         if (type >= RTE_ETH_EVENT_MAX) {
1825                 fprintf(stderr, "\nPort %" PRIu8 ": %s called upon invalid event %d\n",
1826                         port_id, __func__, type);
1827                 fflush(stderr);
1828         } else if (event_print_mask & (UINT32_C(1) << type)) {
1829                 printf("\nPort %" PRIu8 ": %s event\n", port_id,
1830                         event_desc[type]);
1831                 fflush(stdout);
1832         }
1833
1834         switch (type) {
1835         case RTE_ETH_EVENT_INTR_RMV:
1836                 if (rte_eal_alarm_set(100000,
1837                                 rmv_event_callback, (void *)(intptr_t)port_id))
1838                         fprintf(stderr, "Could not set up deferred device removal\n");
1839                 break;
1840         default:
1841                 break;
1842         }
1843 }
1844
1845 static int
1846 set_tx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port)
1847 {
1848         uint16_t i;
1849         int diag;
1850         uint8_t mapping_found = 0;
1851
1852         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
1853                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
1854                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
1855                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
1856                                         tx_queue_stats_mappings[i].queue_id,
1857                                         tx_queue_stats_mappings[i].stats_counter_id);
1858                         if (diag != 0)
1859                                 return diag;
1860                         mapping_found = 1;
1861                 }
1862         }
1863         if (mapping_found)
1864                 port->tx_queue_stats_mapping_enabled = 1;
1865         return 0;
1866 }
1867
1868 static int
1869 set_rx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port)
1870 {
1871         uint16_t i;
1872         int diag;
1873         uint8_t mapping_found = 0;
1874
1875         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
1876                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
1877                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
1878                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
1879                                         rx_queue_stats_mappings[i].queue_id,
1880                                         rx_queue_stats_mappings[i].stats_counter_id);
1881                         if (diag != 0)
1882                                 return diag;
1883                         mapping_found = 1;
1884                 }
1885         }
1886         if (mapping_found)
1887                 port->rx_queue_stats_mapping_enabled = 1;
1888         return 0;
1889 }
1890
1891 static void
1892 map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port)
1893 {
1894         int diag = 0;
1895
1896         diag = set_tx_queue_stats_mapping_registers(pi, port);
1897         if (diag != 0) {
1898                 if (diag == -ENOTSUP) {
1899                         port->tx_queue_stats_mapping_enabled = 0;
1900                         printf("TX queue stats mapping not supported port id=%d\n", pi);
1901                 }
1902                 else
1903                         rte_exit(EXIT_FAILURE,
1904                                         "set_tx_queue_stats_mapping_registers "
1905                                         "failed for port id=%d diag=%d\n",
1906                                         pi, diag);
1907         }
1908
1909         diag = set_rx_queue_stats_mapping_registers(pi, port);
1910         if (diag != 0) {
1911                 if (diag == -ENOTSUP) {
1912                         port->rx_queue_stats_mapping_enabled = 0;
1913                         printf("RX queue stats mapping not supported port id=%d\n", pi);
1914                 }
1915                 else
1916                         rte_exit(EXIT_FAILURE,
1917                                         "set_rx_queue_stats_mapping_registers "
1918                                         "failed for port id=%d diag=%d\n",
1919                                         pi, diag);
1920         }
1921 }
1922
1923 static void
1924 rxtx_port_config(struct rte_port *port)
1925 {
1926         port->rx_conf = port->dev_info.default_rxconf;
1927         port->tx_conf = port->dev_info.default_txconf;
1928
1929         /* Check if any RX/TX parameters have been passed */
1930         if (rx_pthresh != RTE_PMD_PARAM_UNSET)
1931                 port->rx_conf.rx_thresh.pthresh = rx_pthresh;
1932
1933         if (rx_hthresh != RTE_PMD_PARAM_UNSET)
1934                 port->rx_conf.rx_thresh.hthresh = rx_hthresh;
1935
1936         if (rx_wthresh != RTE_PMD_PARAM_UNSET)
1937                 port->rx_conf.rx_thresh.wthresh = rx_wthresh;
1938
1939         if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
1940                 port->rx_conf.rx_free_thresh = rx_free_thresh;
1941
1942         if (rx_drop_en != RTE_PMD_PARAM_UNSET)
1943                 port->rx_conf.rx_drop_en = rx_drop_en;
1944
1945         if (tx_pthresh != RTE_PMD_PARAM_UNSET)
1946                 port->tx_conf.tx_thresh.pthresh = tx_pthresh;
1947
1948         if (tx_hthresh != RTE_PMD_PARAM_UNSET)
1949                 port->tx_conf.tx_thresh.hthresh = tx_hthresh;
1950
1951         if (tx_wthresh != RTE_PMD_PARAM_UNSET)
1952                 port->tx_conf.tx_thresh.wthresh = tx_wthresh;
1953
1954         if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
1955                 port->tx_conf.tx_rs_thresh = tx_rs_thresh;
1956
1957         if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
1958                 port->tx_conf.tx_free_thresh = tx_free_thresh;
1959
1960         if (txq_flags != RTE_PMD_PARAM_UNSET)
1961                 port->tx_conf.txq_flags = txq_flags;
1962 }
1963
1964 void
1965 init_port_config(void)
1966 {
1967         portid_t pid;
1968         struct rte_port *port;
1969
1970         RTE_ETH_FOREACH_DEV(pid) {
1971                 port = &ports[pid];
1972                 port->dev_conf.rxmode = rx_mode;
1973                 port->dev_conf.fdir_conf = fdir_conf;
1974                 if (nb_rxq > 1) {
1975                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1976                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = rss_hf;
1977                 } else {
1978                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1979                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
1980                 }
1981
1982                 if (port->dcb_flag == 0) {
1983                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
1984                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
1985                         else
1986                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
1987                 }
1988
1989                 rxtx_port_config(port);
1990
1991                 rte_eth_macaddr_get(pid, &port->eth_addr);
1992
1993                 map_port_queue_stats_mapping_registers(pid, port);
1994 #ifdef RTE_NIC_BYPASS
1995                 rte_eth_dev_bypass_init(pid);
1996 #endif
1997
1998                 if (lsc_interrupt &&
1999                     (rte_eth_devices[pid].data->dev_flags &
2000                      RTE_ETH_DEV_INTR_LSC))
2001                         port->dev_conf.intr_conf.lsc = 1;
2002                 if (rmv_interrupt &&
2003                     (rte_eth_devices[pid].data->dev_flags &
2004                      RTE_ETH_DEV_INTR_RMV))
2005                         port->dev_conf.intr_conf.rmv = 1;
2006         }
2007 }
2008
2009 void set_port_slave_flag(portid_t slave_pid)
2010 {
2011         struct rte_port *port;
2012
2013         port = &ports[slave_pid];
2014         port->slave_flag = 1;
2015 }
2016
2017 void clear_port_slave_flag(portid_t slave_pid)
2018 {
2019         struct rte_port *port;
2020
2021         port = &ports[slave_pid];
2022         port->slave_flag = 0;
2023 }
2024
2025 uint8_t port_is_bonding_slave(portid_t slave_pid)
2026 {
2027         struct rte_port *port;
2028
2029         port = &ports[slave_pid];
2030         return port->slave_flag;
2031 }
2032
2033 const uint16_t vlan_tags[] = {
2034                 0,  1,  2,  3,  4,  5,  6,  7,
2035                 8,  9, 10, 11,  12, 13, 14, 15,
2036                 16, 17, 18, 19, 20, 21, 22, 23,
2037                 24, 25, 26, 27, 28, 29, 30, 31
2038 };
2039
2040 static  int
2041 get_eth_dcb_conf(struct rte_eth_conf *eth_conf,
2042                  enum dcb_mode_enable dcb_mode,
2043                  enum rte_eth_nb_tcs num_tcs,
2044                  uint8_t pfc_en)
2045 {
2046         uint8_t i;
2047
2048         /*
2049          * Builds up the correct configuration for dcb+vt based on the vlan tags array
2050          * given above, and the number of traffic classes available for use.
2051          */
2052         if (dcb_mode == DCB_VT_ENABLED) {
2053                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
2054                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
2055                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2056                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
2057
2058                 /* VMDQ+DCB RX and TX configurations */
2059                 vmdq_rx_conf->enable_default_pool = 0;
2060                 vmdq_rx_conf->default_pool = 0;
2061                 vmdq_rx_conf->nb_queue_pools =
2062                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2063                 vmdq_tx_conf->nb_queue_pools =
2064                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2065
2066                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
2067                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
2068                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
2069                         vmdq_rx_conf->pool_map[i].pools =
2070                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
2071                 }
2072                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2073                         vmdq_rx_conf->dcb_tc[i] = i;
2074                         vmdq_tx_conf->dcb_tc[i] = i;
2075                 }
2076
2077                 /* set DCB mode of RX and TX of multiple queues */
2078                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
2079                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
2080         } else {
2081                 struct rte_eth_dcb_rx_conf *rx_conf =
2082                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
2083                 struct rte_eth_dcb_tx_conf *tx_conf =
2084                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
2085
2086                 rx_conf->nb_tcs = num_tcs;
2087                 tx_conf->nb_tcs = num_tcs;
2088
2089                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2090                         rx_conf->dcb_tc[i] = i % num_tcs;
2091                         tx_conf->dcb_tc[i] = i % num_tcs;
2092                 }
2093                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
2094                 eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf;
2095                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
2096         }
2097
2098         if (pfc_en)
2099                 eth_conf->dcb_capability_en =
2100                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
2101         else
2102                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
2103
2104         return 0;
2105 }
2106
2107 int
2108 init_port_dcb_config(portid_t pid,
2109                      enum dcb_mode_enable dcb_mode,
2110                      enum rte_eth_nb_tcs num_tcs,
2111                      uint8_t pfc_en)
2112 {
2113         struct rte_eth_conf port_conf;
2114         struct rte_port *rte_port;
2115         int retval;
2116         uint16_t i;
2117
2118         rte_port = &ports[pid];
2119
2120         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
2121         /* Enter DCB configuration status */
2122         dcb_config = 1;
2123
2124         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
2125         retval = get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en);
2126         if (retval < 0)
2127                 return retval;
2128         port_conf.rxmode.hw_vlan_filter = 1;
2129
2130         /**
2131          * Write the configuration into the device.
2132          * Set the numbers of RX & TX queues to 0, so
2133          * the RX & TX queues will not be setup.
2134          */
2135         (void)rte_eth_dev_configure(pid, 0, 0, &port_conf);
2136
2137         rte_eth_dev_info_get(pid, &rte_port->dev_info);
2138
2139         /* If dev_info.vmdq_pool_base is greater than 0,
2140          * the queue id of vmdq pools is started after pf queues.
2141          */
2142         if (dcb_mode == DCB_VT_ENABLED &&
2143             rte_port->dev_info.vmdq_pool_base > 0) {
2144                 printf("VMDQ_DCB multi-queue mode is nonsensical"
2145                         " for port %d.", pid);
2146                 return -1;
2147         }
2148
2149         /* Assume the ports in testpmd have the same dcb capability
2150          * and has the same number of rxq and txq in dcb mode
2151          */
2152         if (dcb_mode == DCB_VT_ENABLED) {
2153                 if (rte_port->dev_info.max_vfs > 0) {
2154                         nb_rxq = rte_port->dev_info.nb_rx_queues;
2155                         nb_txq = rte_port->dev_info.nb_tx_queues;
2156                 } else {
2157                         nb_rxq = rte_port->dev_info.max_rx_queues;
2158                         nb_txq = rte_port->dev_info.max_tx_queues;
2159                 }
2160         } else {
2161                 /*if vt is disabled, use all pf queues */
2162                 if (rte_port->dev_info.vmdq_pool_base == 0) {
2163                         nb_rxq = rte_port->dev_info.max_rx_queues;
2164                         nb_txq = rte_port->dev_info.max_tx_queues;
2165                 } else {
2166                         nb_rxq = (queueid_t)num_tcs;
2167                         nb_txq = (queueid_t)num_tcs;
2168
2169                 }
2170         }
2171         rx_free_thresh = 64;
2172
2173         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
2174
2175         rxtx_port_config(rte_port);
2176         /* VLAN filter */
2177         rte_port->dev_conf.rxmode.hw_vlan_filter = 1;
2178         for (i = 0; i < RTE_DIM(vlan_tags); i++)
2179                 rx_vft_set(pid, vlan_tags[i], 1);
2180
2181         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
2182         map_port_queue_stats_mapping_registers(pid, rte_port);
2183
2184         rte_port->dcb_flag = 1;
2185
2186         return 0;
2187 }
2188
2189 static void
2190 init_port(void)
2191 {
2192         /* Configuration of Ethernet ports. */
2193         ports = rte_zmalloc("testpmd: ports",
2194                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
2195                             RTE_CACHE_LINE_SIZE);
2196         if (ports == NULL) {
2197                 rte_exit(EXIT_FAILURE,
2198                                 "rte_zmalloc(%d struct rte_port) failed\n",
2199                                 RTE_MAX_ETHPORTS);
2200         }
2201 }
2202
2203 static void
2204 force_quit(void)
2205 {
2206         pmd_test_exit();
2207         prompt_exit();
2208 }
2209
2210 static void
2211 signal_handler(int signum)
2212 {
2213         if (signum == SIGINT || signum == SIGTERM) {
2214                 printf("\nSignal %d received, preparing to exit...\n",
2215                                 signum);
2216 #ifdef RTE_LIBRTE_PDUMP
2217                 /* uninitialize packet capture framework */
2218                 rte_pdump_uninit();
2219 #endif
2220 #ifdef RTE_LIBRTE_LATENCY_STATS
2221                 rte_latencystats_uninit();
2222 #endif
2223                 force_quit();
2224                 /* exit with the expected status */
2225                 signal(signum, SIG_DFL);
2226                 kill(getpid(), signum);
2227         }
2228 }
2229
2230 int
2231 main(int argc, char** argv)
2232 {
2233         int  diag;
2234         uint8_t port_id;
2235
2236         signal(SIGINT, signal_handler);
2237         signal(SIGTERM, signal_handler);
2238
2239         diag = rte_eal_init(argc, argv);
2240         if (diag < 0)
2241                 rte_panic("Cannot init EAL\n");
2242
2243 #ifdef RTE_LIBRTE_PDUMP
2244         /* initialize packet capture framework */
2245         rte_pdump_init(NULL);
2246 #endif
2247
2248         nb_ports = (portid_t) rte_eth_dev_count();
2249         if (nb_ports == 0)
2250                 RTE_LOG(WARNING, EAL, "No probed ethernet devices\n");
2251
2252         /* allocate port structures, and init them */
2253         init_port();
2254
2255         set_def_fwd_config();
2256         if (nb_lcores == 0)
2257                 rte_panic("Empty set of forwarding logical cores - check the "
2258                           "core mask supplied in the command parameters\n");
2259
2260         /* Bitrate/latency stats disabled by default */
2261 #ifdef RTE_LIBRTE_BITRATE
2262         bitrate_enabled = 0;
2263 #endif
2264 #ifdef RTE_LIBRTE_LATENCY_STATS
2265         latencystats_enabled = 0;
2266 #endif
2267
2268         argc -= diag;
2269         argv += diag;
2270         if (argc > 1)
2271                 launch_args_parse(argc, argv);
2272
2273         if (!nb_rxq && !nb_txq)
2274                 printf("Warning: Either rx or tx queues should be non-zero\n");
2275
2276         if (nb_rxq > 1 && nb_rxq > nb_txq)
2277                 printf("Warning: nb_rxq=%d enables RSS configuration, "
2278                        "but nb_txq=%d will prevent to fully test it.\n",
2279                        nb_rxq, nb_txq);
2280
2281         init_config();
2282         if (start_port(RTE_PORT_ALL) != 0)
2283                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
2284
2285         /* set all ports to promiscuous mode by default */
2286         RTE_ETH_FOREACH_DEV(port_id)
2287                 rte_eth_promiscuous_enable(port_id);
2288
2289         /* Init metrics library */
2290         rte_metrics_init(rte_socket_id());
2291
2292 #ifdef RTE_LIBRTE_LATENCY_STATS
2293         if (latencystats_enabled != 0) {
2294                 int ret = rte_latencystats_init(1, NULL);
2295                 if (ret)
2296                         printf("Warning: latencystats init()"
2297                                 " returned error %d\n", ret);
2298                 printf("Latencystats running on lcore %d\n",
2299                         latencystats_lcore_id);
2300         }
2301 #endif
2302
2303         /* Setup bitrate stats */
2304 #ifdef RTE_LIBRTE_BITRATE
2305         if (bitrate_enabled != 0) {
2306                 bitrate_data = rte_stats_bitrate_create();
2307                 if (bitrate_data == NULL)
2308                         rte_exit(EXIT_FAILURE,
2309                                 "Could not allocate bitrate data.\n");
2310                 rte_stats_bitrate_reg(bitrate_data);
2311         }
2312 #endif
2313
2314 #ifdef RTE_LIBRTE_CMDLINE
2315         if (strlen(cmdline_filename) != 0)
2316                 cmdline_read_from_file(cmdline_filename);
2317
2318         if (interactive == 1) {
2319                 if (auto_start) {
2320                         printf("Start automatic packet forwarding\n");
2321                         start_packet_forwarding(0);
2322                 }
2323                 prompt();
2324                 pmd_test_exit();
2325         } else
2326 #endif
2327         {
2328                 char c;
2329                 int rc;
2330
2331                 printf("No commandline core given, start packet forwarding\n");
2332                 start_packet_forwarding(0);
2333                 printf("Press enter to exit\n");
2334                 rc = read(0, &c, 1);
2335                 pmd_test_exit();
2336                 if (rc < 0)
2337                         return 1;
2338         }
2339
2340         return 0;
2341 }