app/test-pmd/testpmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdarg.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <signal.h>
  38 #include <string.h>
  39 #include <time.h>
  40 #include <fcntl.h>
  41 #include <sys/types.h>
  42 #include <errno.h>
  43
  44 #include <sys/queue.h>
  45 #include <sys/stat.h>
  46
  47 #include <stdint.h>
  48 #include <unistd.h>
  49 #include <inttypes.h>
  50
  51 #include <rte_common.h>
  52 #include <rte_errno.h>
  53 #include <rte_byteorder.h>
  54 #include <rte_log.h>
  55 #include <rte_debug.h>
  56 #include <rte_cycles.h>
  57 #include <rte_memory.h>
  58 #include <rte_memcpy.h>
  59 #include <rte_memzone.h>
  60 #include <rte_launch.h>
  61 #include <rte_eal.h>
  62 #include <rte_alarm.h>
  63 #include <rte_per_lcore.h>
  64 #include <rte_lcore.h>
  65 #include <rte_atomic.h>
  66 #include <rte_branch_prediction.h>
  67 #include <rte_mempool.h>
  68 #include <rte_malloc.h>
  69 #include <rte_mbuf.h>
  70 #include <rte_interrupts.h>
  71 #include <rte_pci.h>
  72 #include <rte_ether.h>
  73 #include <rte_ethdev.h>
  74 #include <rte_dev.h>
  75 #include <rte_string_fns.h>
  76 #ifdef RTE_LIBRTE_PMD_XENVIRT
  77 #include <rte_eth_xenvirt.h>
  78 #endif
  79 #ifdef RTE_LIBRTE_PDUMP
  80 #include <rte_pdump.h>
  81 #endif
  82 #include <rte_flow.h>
  83 #include <rte_metrics.h>
  84 #ifdef RTE_LIBRTE_BITRATE
  85 #include <rte_bitrate.h>
  86 #endif
  87 #ifdef RTE_LIBRTE_LATENCY_STATS
  88 #include <rte_latencystats.h>
  89 #endif
  90
  91 #include "testpmd.h"
  92
  93 uint16_t verbose_level = 0; /**< Silent by default. */
  94
  95 /* use master core for command line ? */
  96 uint8_t interactive = 0;
  97 uint8_t auto_start = 0;
  98 char cmdline_filename[PATH_MAX] = {0};
  99
 100 /*
 101  * NUMA support configuration.
 102  * When set, the NUMA support attempts to dispatch the allocation of the
 103  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 104  * probed ports among the CPU sockets 0 and 1.
 105  * Otherwise, all memory is allocated from CPU socket 0.
 106  */
 107 uint8_t numa_support = 1; /**< numa enabled by default */
 108
 109 /*
 110  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 111  * not configured.
 112  */
 113 uint8_t socket_num = UMA_NO_CONFIG;
 114
 115 /*
 116  * Use ANONYMOUS mapped memory (might be not physically continuous) for mbufs.
 117  */
 118 uint8_t mp_anon = 0;
 119
 120 /*
 121  * Record the Ethernet address of peer target ports to which packets are
 122  * forwarded.
 123  * Must be instantiated with the ethernet addresses of peer traffic generator
 124  * ports.
 125  */
 126 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 127 portid_t nb_peer_eth_addrs = 0;
 128
 129 /*
 130  * Probed Target Environment.
 131  */
 132 struct rte_port *ports;        /**< For all probed ethernet ports. */
 133 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 134 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 135 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 136
 137 /*
 138  * Test Forwarding Configuration.
 139  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 140  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 141  */
 142 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 143 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 144 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 145 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 146
 147 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 148 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 149
 150 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 151 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 152
 153 /*
 154  * Forwarding engines.
 155  */
 156 struct fwd_engine * fwd_engines[] = {
 157         &io_fwd_engine,
 158         &mac_fwd_engine,
 159         &mac_swap_engine,
 160         &flow_gen_engine,
 161         &rx_only_engine,
 162         &tx_only_engine,
 163         &csum_fwd_engine,
 164         &icmp_echo_engine,
 165 #ifdef RTE_LIBRTE_IEEE1588
 166         &ieee1588_fwd_engine,
 167 #endif
 168         NULL,
 169 };
 170
 171 struct fwd_config cur_fwd_config;
 172 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 173 uint32_t retry_enabled;
 174 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 175 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 176
 177 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 178 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 179                                       * specified on command-line. */
 180
 181 /*
 182  * Configuration of packet segments used by the "txonly" processing engine.
 183  */
 184 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 185 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 186         TXONLY_DEF_PACKET_LEN,
 187 };
 188 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 189
 190 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 191 /**< Split policy for packets to TX. */
 192
 193 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 194 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 195
 196 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 197 uint8_t dcb_config = 0;
 198
 199 /* Whether the dcb is in testing status */
 200 uint8_t dcb_test = 0;
 201
 202 /*
 203  * Configurable number of RX/TX queues.
 204  */
 205 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 206 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 207
 208 /*
 209  * Configurable number of RX/TX ring descriptors.
 210  */
 211 #define RTE_TEST_RX_DESC_DEFAULT 128
 212 #define RTE_TEST_TX_DESC_DEFAULT 512
 213 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 214 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 215
 216 #define RTE_PMD_PARAM_UNSET -1
 217 /*
 218  * Configurable values of RX and TX ring threshold registers.
 219  */
 220
 221 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 222 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 223 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 224
 225 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 226 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 227 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 228
 229 /*
 230  * Configurable value of RX free threshold.
 231  */
 232 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 233
 234 /*
 235  * Configurable value of RX drop enable.
 236  */
 237 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 238
 239 /*
 240  * Configurable value of TX free threshold.
 241  */
 242 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 243
 244 /*
 245  * Configurable value of TX RS bit threshold.
 246  */
 247 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 248
 249 /*
 250  * Configurable value of TX queue flags.
 251  */
 252 int32_t txq_flags = RTE_PMD_PARAM_UNSET;
 253
 254 /*
 255  * Receive Side Scaling (RSS) configuration.
 256  */
 257 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 258
 259 /*
 260  * Port topology configuration
 261  */
 262 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 263
 264 /*
 265  * Avoids to flush all the RX streams before starts forwarding.
 266  */
 267 uint8_t no_flush_rx = 0; /* flush by default */
 268
 269 /*
 270  * Avoids to check link status when starting/stopping a port.
 271  */
 272 uint8_t no_link_check = 0; /* check by default */
 273
 274 /*
 275  * Enable link status change notification
 276  */
 277 uint8_t lsc_interrupt = 1; /* enabled by default */
 278
 279 /*
 280  * Enable device removal notification.
 281  */
 282 uint8_t rmv_interrupt = 1; /* enabled by default */
 283
 284 /*
 285  * NIC bypass mode configuration options.
 286  */
 287 #ifdef RTE_NIC_BYPASS
 288
 289 /* The NIC bypass watchdog timeout. */
 290 uint32_t bypass_timeout = RTE_BYPASS_TMT_OFF;
 291
 292 #endif
 293
 294 #ifdef RTE_LIBRTE_LATENCY_STATS
 295
 296 /*
 297  * Set when latency stats is enabled in the commandline
 298  */
 299 uint8_t latencystats_enabled;
 300
 301 /*
 302  * Lcore ID to serive latency statistics.
 303  */
 304 lcoreid_t latencystats_lcore_id = -1;
 305
 306 #endif
 307
 308 /*
 309  * Ethernet device configuration.
 310  */
 311 struct rte_eth_rxmode rx_mode = {
 312         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 313         .split_hdr_size = 0,
 314         .header_split   = 0, /**< Header Split disabled. */
 315         .hw_ip_checksum = 0, /**< IP checksum offload disabled. */
 316         .hw_vlan_filter = 1, /**< VLAN filtering enabled. */
 317         .hw_vlan_strip  = 1, /**< VLAN strip enabled. */
 318         .hw_vlan_extend = 0, /**< Extended VLAN disabled. */
 319         .jumbo_frame    = 0, /**< Jumbo Frame Support disabled. */
 320         .hw_strip_crc   = 1, /**< CRC stripping by hardware enabled. */
 321 };
 322
 323 struct rte_fdir_conf fdir_conf = {
 324         .mode = RTE_FDIR_MODE_NONE,
 325         .pballoc = RTE_FDIR_PBALLOC_64K,
 326         .status = RTE_FDIR_REPORT_STATUS,
 327         .mask = {
 328                 .vlan_tci_mask = 0x0,
 329                 .ipv4_mask     = {
 330                         .src_ip = 0xFFFFFFFF,
 331                         .dst_ip = 0xFFFFFFFF,
 332                 },
 333                 .ipv6_mask     = {
 334                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 335                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 336                 },
 337                 .src_port_mask = 0xFFFF,
 338                 .dst_port_mask = 0xFFFF,
 339                 .mac_addr_byte_mask = 0xFF,
 340                 .tunnel_type_mask = 1,
 341                 .tunnel_id_mask = 0xFFFFFFFF,
 342         },
 343         .drop_queue = 127,
 344 };
 345
 346 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 347
 348 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 349 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 350
 351 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 352 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 353
 354 uint16_t nb_tx_queue_stats_mappings = 0;
 355 uint16_t nb_rx_queue_stats_mappings = 0;
 356
 357 unsigned max_socket = 0;
 358
 359 #ifdef RTE_LIBRTE_BITRATE
 360 /* Bitrate statistics */
 361 struct rte_stats_bitrates *bitrate_data;
 362 lcoreid_t bitrate_lcore_id;
 363 uint8_t bitrate_enabled;
 364 #endif
 365
 366 /* Forward function declarations */
 367 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port);
 368 static void check_all_ports_link_status(uint32_t port_mask);
 369 static void eth_event_callback(uint8_t port_id,
 370                                enum rte_eth_event_type type,
 371                                void *param);
 372
 373 /*
 374  * Check if all the ports are started.
 375  * If yes, return positive value. If not, return zero.
 376  */
 377 static int all_ports_started(void);
 378
 379 /*
 380  * Setup default configuration.
 381  */
 382 static void
 383 set_default_fwd_lcores_config(void)
 384 {
 385         unsigned int i;
 386         unsigned int nb_lc;
 387         unsigned int sock_num;
 388
 389         nb_lc = 0;
 390         for (i = 0; i < RTE_MAX_LCORE; i++) {
 391                 sock_num = rte_lcore_to_socket_id(i) + 1;
 392                 if (sock_num > max_socket) {
 393                         if (sock_num > RTE_MAX_NUMA_NODES)
 394                                 rte_exit(EXIT_FAILURE, "Total sockets greater than %u\n", RTE_MAX_NUMA_NODES);
 395                         max_socket = sock_num;
 396                 }
 397                 if (!rte_lcore_is_enabled(i))
 398                         continue;
 399                 if (i == rte_get_master_lcore())
 400                         continue;
 401                 fwd_lcores_cpuids[nb_lc++] = i;
 402         }
 403         nb_lcores = (lcoreid_t) nb_lc;
 404         nb_cfg_lcores = nb_lcores;
 405         nb_fwd_lcores = 1;
 406 }
 407
 408 static void
 409 set_def_peer_eth_addrs(void)
 410 {
 411         portid_t i;
 412
 413         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 414                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 415                 peer_eth_addrs[i].addr_bytes[5] = i;
 416         }
 417 }
 418
 419 static void
 420 set_default_fwd_ports_config(void)
 421 {
 422         portid_t pt_id;
 423
 424         for (pt_id = 0; pt_id < nb_ports; pt_id++)
 425                 fwd_ports_ids[pt_id] = pt_id;
 426
 427         nb_cfg_ports = nb_ports;
 428         nb_fwd_ports = nb_ports;
 429 }
 430
 431 void
 432 set_def_fwd_config(void)
 433 {
 434         set_default_fwd_lcores_config();
 435         set_def_peer_eth_addrs();
 436         set_default_fwd_ports_config();
 437 }
 438
 439 /*
 440  * Configuration initialisation done once at init time.
 441  */
 442 static void
 443 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 444                  unsigned int socket_id)
 445 {
 446         char pool_name[RTE_MEMPOOL_NAMESIZE];
 447         struct rte_mempool *rte_mp = NULL;
 448         uint32_t mb_size;
 449
 450         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 451         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 452
 453         RTE_LOG(INFO, USER1,
 454                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 455                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 456
 457 #ifdef RTE_LIBRTE_PMD_XENVIRT
 458         rte_mp = rte_mempool_gntalloc_create(pool_name, nb_mbuf, mb_size,
 459                 (unsigned) mb_mempool_cache,
 460                 sizeof(struct rte_pktmbuf_pool_private),
 461                 rte_pktmbuf_pool_init, NULL,
 462                 rte_pktmbuf_init, NULL,
 463                 socket_id, 0);
 464 #endif
 465
 466         /* if the former XEN allocation failed fall back to normal allocation */
 467         if (rte_mp == NULL) {
 468                 if (mp_anon != 0) {
 469                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 470                                 mb_size, (unsigned) mb_mempool_cache,
 471                                 sizeof(struct rte_pktmbuf_pool_private),
 472                                 socket_id, 0);
 473                         if (rte_mp == NULL)
 474                                 goto err;
 475
 476                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 477                                 rte_mempool_free(rte_mp);
 478                                 rte_mp = NULL;
 479                                 goto err;
 480                         }
 481                         rte_pktmbuf_pool_init(rte_mp, NULL);
 482                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 483                 } else {
 484                         /* wrapper to rte_mempool_create() */
 485                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 486                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 487                 }
 488         }
 489
 490 err:
 491         if (rte_mp == NULL) {
 492                 rte_exit(EXIT_FAILURE,
 493                         "Creation of mbuf pool for socket %u failed: %s\n",
 494                         socket_id, rte_strerror(rte_errno));
 495         } else if (verbose_level > 0) {
 496                 rte_mempool_dump(stdout, rte_mp);
 497         }
 498 }
 499
 500 /*
 501  * Check given socket id is valid or not with NUMA mode,
 502  * if valid, return 0, else return -1
 503  */
 504 static int
 505 check_socket_id(const unsigned int socket_id)
 506 {
 507         static int warning_once = 0;
 508
 509         if (socket_id >= max_socket) {
 510                 if (!warning_once && numa_support)
 511                         printf("Warning: NUMA should be configured manually by"
 512                                " using --port-numa-config and"
 513                                " --ring-numa-config parameters along with"
 514                                " --numa.\n");
 515                 warning_once = 1;
 516                 return -1;
 517         }
 518         return 0;
 519 }
 520
 521 static void
 522 init_config(void)
 523 {
 524         portid_t pid;
 525         struct rte_port *port;
 526         struct rte_mempool *mbp;
 527         unsigned int nb_mbuf_per_pool;
 528         lcoreid_t  lc_id;
 529         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
 530
 531         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
 532         /* Configuration of logical cores. */
 533         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
 534                                 sizeof(struct fwd_lcore *) * nb_lcores,
 535                                 RTE_CACHE_LINE_SIZE);
 536         if (fwd_lcores == NULL) {
 537                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
 538                                                         "failed\n", nb_lcores);
 539         }
 540         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 541                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
 542                                                sizeof(struct fwd_lcore),
 543                                                RTE_CACHE_LINE_SIZE);
 544                 if (fwd_lcores[lc_id] == NULL) {
 545                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
 546                                                                 "failed\n");
 547                 }
 548                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
 549         }
 550
 551         RTE_ETH_FOREACH_DEV(pid) {
 552                 port = &ports[pid];
 553                 rte_eth_dev_info_get(pid, &port->dev_info);
 554
 555                 if (numa_support) {
 556                         if (port_numa[pid] != NUMA_NO_CONFIG)
 557                                 port_per_socket[port_numa[pid]]++;
 558                         else {
 559                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
 560
 561                                 /* if socket_id is invalid, set to 0 */
 562                                 if (check_socket_id(socket_id) < 0)
 563                                         socket_id = 0;
 564                                 port_per_socket[socket_id]++;
 565                         }
 566                 }
 567
 568                 /* set flag to initialize port/queue */
 569                 port->need_reconfig = 1;
 570                 port->need_reconfig_queues = 1;
 571         }
 572
 573         /*
 574          * Create pools of mbuf.
 575          * If NUMA support is disabled, create a single pool of mbuf in
 576          * socket 0 memory by default.
 577          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
 578          *
 579          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
 580          * nb_txd can be configured at run time.
 581          */
 582         if (param_total_num_mbufs)
 583                 nb_mbuf_per_pool = param_total_num_mbufs;
 584         else {
 585                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
 586                         (nb_lcores * mb_mempool_cache) +
 587                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
 588                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
 589         }
 590
 591         if (numa_support) {
 592                 uint8_t i;
 593
 594                 for (i = 0; i < max_socket; i++)
 595                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, i);
 596         } else {
 597                 if (socket_num == UMA_NO_CONFIG)
 598                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0);
 599                 else
 600                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
 601                                                  socket_num);
 602         }
 603
 604         init_port_config();
 605
 606         /*
 607          * Records which Mbuf pool to use by each logical core, if needed.
 608          */
 609         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 610                 mbp = mbuf_pool_find(
 611                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
 612
 613                 if (mbp == NULL)
 614                         mbp = mbuf_pool_find(0);
 615                 fwd_lcores[lc_id]->mbp = mbp;
 616         }
 617
 618         /* Configuration of packet forwarding streams. */
 619         if (init_fwd_streams() < 0)
 620                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 621
 622         fwd_config_setup();
 623 }
 624
 625
 626 void
 627 reconfig(portid_t new_port_id, unsigned socket_id)
 628 {
 629         struct rte_port *port;
 630
 631         /* Reconfiguration of Ethernet ports. */
 632         port = &ports[new_port_id];
 633         rte_eth_dev_info_get(new_port_id, &port->dev_info);
 634
 635         /* set flag to initialize port/queue */
 636         port->need_reconfig = 1;
 637         port->need_reconfig_queues = 1;
 638         port->socket_id = socket_id;
 639
 640         init_port_config();
 641 }
 642
 643
 644 int
 645 init_fwd_streams(void)
 646 {
 647         portid_t pid;
 648         struct rte_port *port;
 649         streamid_t sm_id, nb_fwd_streams_new;
 650         queueid_t q;
 651
 652         /* set socket id according to numa or not */
 653         RTE_ETH_FOREACH_DEV(pid) {
 654                 port = &ports[pid];
 655                 if (nb_rxq > port->dev_info.max_rx_queues) {
 656                         printf("Fail: nb_rxq(%d) is greater than "
 657                                 "max_rx_queues(%d)\n", nb_rxq,
 658                                 port->dev_info.max_rx_queues);
 659                         return -1;
 660                 }
 661                 if (nb_txq > port->dev_info.max_tx_queues) {
 662                         printf("Fail: nb_txq(%d) is greater than "
 663                                 "max_tx_queues(%d)\n", nb_txq,
 664                                 port->dev_info.max_tx_queues);
 665                         return -1;
 666                 }
 667                 if (numa_support) {
 668                         if (port_numa[pid] != NUMA_NO_CONFIG)
 669                                 port->socket_id = port_numa[pid];
 670                         else {
 671                                 port->socket_id = rte_eth_dev_socket_id(pid);
 672
 673                                 /* if socket_id is invalid, set to 0 */
 674                                 if (check_socket_id(port->socket_id) < 0)
 675                                         port->socket_id = 0;
 676                         }
 677                 }
 678                 else {
 679                         if (socket_num == UMA_NO_CONFIG)
 680                                 port->socket_id = 0;
 681                         else
 682                                 port->socket_id = socket_num;
 683                 }
 684         }
 685
 686         q = RTE_MAX(nb_rxq, nb_txq);
 687         if (q == 0) {
 688                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
 689                 return -1;
 690         }
 691         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
 692         if (nb_fwd_streams_new == nb_fwd_streams)
 693                 return 0;
 694         /* clear the old */
 695         if (fwd_streams != NULL) {
 696                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 697                         if (fwd_streams[sm_id] == NULL)
 698                                 continue;
 699                         rte_free(fwd_streams[sm_id]);
 700                         fwd_streams[sm_id] = NULL;
 701                 }
 702                 rte_free(fwd_streams);
 703                 fwd_streams = NULL;
 704         }
 705
 706         /* init new */
 707         nb_fwd_streams = nb_fwd_streams_new;
 708         fwd_streams = rte_zmalloc("testpmd: fwd_streams",
 709                 sizeof(struct fwd_stream *) * nb_fwd_streams, RTE_CACHE_LINE_SIZE);
 710         if (fwd_streams == NULL)
 711                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_stream *)) "
 712                                                 "failed\n", nb_fwd_streams);
 713
 714         for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 715                 fwd_streams[sm_id] = rte_zmalloc("testpmd: struct fwd_stream",
 716                                 sizeof(struct fwd_stream), RTE_CACHE_LINE_SIZE);
 717                 if (fwd_streams[sm_id] == NULL)
 718                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_stream)"
 719                                                                 " failed\n");
 720         }
 721
 722         return 0;
 723 }
 724
 725 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 726 static void
 727 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
 728 {
 729         unsigned int total_burst;
 730         unsigned int nb_burst;
 731         unsigned int burst_stats[3];
 732         uint16_t pktnb_stats[3];
 733         uint16_t nb_pkt;
 734         int burst_percent[3];
 735
 736         /*
 737          * First compute the total number of packet bursts and the
 738          * two highest numbers of bursts of the same number of packets.
 739          */
 740         total_burst = 0;
 741         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
 742         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
 743         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
 744                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
 745                 if (nb_burst == 0)
 746                         continue;
 747                 total_burst += nb_burst;
 748                 if (nb_burst > burst_stats[0]) {
 749                         burst_stats[1] = burst_stats[0];
 750                         pktnb_stats[1] = pktnb_stats[0];
 751                         burst_stats[0] = nb_burst;
 752                         pktnb_stats[0] = nb_pkt;
 753                 }
 754         }
 755         if (total_burst == 0)
 756                 return;
 757         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
 758         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
 759                burst_percent[0], (int) pktnb_stats[0]);
 760         if (burst_stats[0] == total_burst) {
 761                 printf("]\n");
 762                 return;
 763         }
 764         if (burst_stats[0] + burst_stats[1] == total_burst) {
 765                 printf(" + %d%% of %d pkts]\n",
 766                        100 - burst_percent[0], pktnb_stats[1]);
 767                 return;
 768         }
 769         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
 770         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
 771         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
 772                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
 773                 return;
 774         }
 775         printf(" + %d%% of %d pkts + %d%% of others]\n",
 776                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
 777 }
 778 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
 779
 780 static void
 781 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats)
 782 {
 783         struct rte_port *port;
 784         uint8_t i;
 785
 786         static const char *fwd_stats_border = "----------------------";
 787
 788         port = &ports[port_id];
 789         printf("\n  %s Forward statistics for port %-2d %s\n",
 790                fwd_stats_border, port_id, fwd_stats_border);
 791
 792         if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) {
 793                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
 794                        "%-"PRIu64"\n",
 795                        stats->ipackets, stats->imissed,
 796                        (uint64_t) (stats->ipackets + stats->imissed));
 797
 798                 if (cur_fwd_eng == &csum_fwd_engine)
 799                         printf("  Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n",
 800                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 801                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
 802                         printf("  RX-error: %-"PRIu64"\n",  stats->ierrors);
 803                         printf("  RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf);
 804                 }
 805
 806                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
 807                        "%-"PRIu64"\n",
 808                        stats->opackets, port->tx_dropped,
 809                        (uint64_t) (stats->opackets + port->tx_dropped));
 810         }
 811         else {
 812                 printf("  RX-packets:             %14"PRIu64"    RX-dropped:%14"PRIu64"    RX-total:"
 813                        "%14"PRIu64"\n",
 814                        stats->ipackets, stats->imissed,
 815                        (uint64_t) (stats->ipackets + stats->imissed));
 816
 817                 if (cur_fwd_eng == &csum_fwd_engine)
 818                         printf("  Bad-ipcsum:%14"PRIu64"    Bad-l4csum:%14"PRIu64"\n",
 819                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 820                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
 821                         printf("  RX-error:%"PRIu64"\n", stats->ierrors);
 822                         printf("  RX-nombufs:             %14"PRIu64"\n",
 823                                stats->rx_nombuf);
 824                 }
 825
 826                 printf("  TX-packets:             %14"PRIu64"    TX-dropped:%14"PRIu64"    TX-total:"
 827                        "%14"PRIu64"\n",
 828                        stats->opackets, port->tx_dropped,
 829                        (uint64_t) (stats->opackets + port->tx_dropped));
 830         }
 831
 832 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 833         if (port->rx_stream)
 834                 pkt_burst_stats_display("RX",
 835                         &port->rx_stream->rx_burst_stats);
 836         if (port->tx_stream)
 837                 pkt_burst_stats_display("TX",
 838                         &port->tx_stream->tx_burst_stats);
 839 #endif
 840
 841         if (port->rx_queue_stats_mapping_enabled) {
 842                 printf("\n");
 843                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 844                         printf("  Stats reg %2d RX-packets:%14"PRIu64
 845                                "     RX-errors:%14"PRIu64
 846                                "    RX-bytes:%14"PRIu64"\n",
 847                                i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]);
 848                 }
 849                 printf("\n");
 850         }
 851         if (port->tx_queue_stats_mapping_enabled) {
 852                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 853                         printf("  Stats reg %2d TX-packets:%14"PRIu64
 854                                "                                 TX-bytes:%14"PRIu64"\n",
 855                                i, stats->q_opackets[i], stats->q_obytes[i]);
 856                 }
 857         }
 858
 859         printf("  %s--------------------------------%s\n",
 860                fwd_stats_border, fwd_stats_border);
 861 }
 862
 863 static void
 864 fwd_stream_stats_display(streamid_t stream_id)
 865 {
 866         struct fwd_stream *fs;
 867         static const char *fwd_top_stats_border = "-------";
 868
 869         fs = fwd_streams[stream_id];
 870         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
 871             (fs->fwd_dropped == 0))
 872                 return;
 873         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
 874                "TX Port=%2d/Queue=%2d %s\n",
 875                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
 876                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
 877         printf("  RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u",
 878                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
 879
 880         /* if checksum mode */
 881         if (cur_fwd_eng == &csum_fwd_engine) {
 882                printf("  RX- bad IP checksum: %-14u  Rx- bad L4 checksum: "
 883                         "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum);
 884         }
 885
 886 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 887         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
 888         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
 889 #endif
 890 }
 891
 892 static void
 893 flush_fwd_rx_queues(void)
 894 {
 895         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 896         portid_t  rxp;
 897         portid_t port_id;
 898         queueid_t rxq;
 899         uint16_t  nb_rx;
 900         uint16_t  i;
 901         uint8_t   j;
 902         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
 903         uint64_t timer_period;
 904
 905         /* convert to number of cycles */
 906         timer_period = rte_get_timer_hz(); /* 1 second timeout */
 907
 908         for (j = 0; j < 2; j++) {
 909                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
 910                         for (rxq = 0; rxq < nb_rxq; rxq++) {
 911                                 port_id = fwd_ports_ids[rxp];
 912                                 /**
 913                                 * testpmd can stuck in the below do while loop
 914                                 * if rte_eth_rx_burst() always returns nonzero
 915                                 * packets. So timer is added to exit this loop
 916                                 * after 1sec timer expiry.
 917                                 */
 918                                 prev_tsc = rte_rdtsc();
 919                                 do {
 920                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
 921                                                 pkts_burst, MAX_PKT_BURST);
 922                                         for (i = 0; i < nb_rx; i++)
 923                                                 rte_pktmbuf_free(pkts_burst[i]);
 924
 925                                         cur_tsc = rte_rdtsc();
 926                                         diff_tsc = cur_tsc - prev_tsc;
 927                                         timer_tsc += diff_tsc;
 928                                 } while ((nb_rx > 0) &&
 929                                         (timer_tsc < timer_period));
 930                                 timer_tsc = 0;
 931                         }
 932                 }
 933                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
 934         }
 935 }
 936
 937 static void
 938 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
 939 {
 940         struct fwd_stream **fsm;
 941         streamid_t nb_fs;
 942         streamid_t sm_id;
 943 #ifdef RTE_LIBRTE_BITRATE
 944         uint64_t tics_per_1sec;
 945         uint64_t tics_datum;
 946         uint64_t tics_current;
 947         uint8_t idx_port, cnt_ports;
 948
 949         cnt_ports = rte_eth_dev_count();
 950         tics_datum = rte_rdtsc();
 951         tics_per_1sec = rte_get_timer_hz();
 952 #endif
 953         fsm = &fwd_streams[fc->stream_idx];
 954         nb_fs = fc->stream_nb;
 955         do {
 956                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
 957                         (*pkt_fwd)(fsm[sm_id]);
 958 #ifdef RTE_LIBRTE_BITRATE
 959                 if (bitrate_enabled != 0 &&
 960                                 bitrate_lcore_id == rte_lcore_id()) {
 961                         tics_current = rte_rdtsc();
 962                         if (tics_current - tics_datum >= tics_per_1sec) {
 963                                 /* Periodic bitrate calculation */
 964                                 for (idx_port = 0;
 965                                                 idx_port < cnt_ports;
 966                                                 idx_port++)
 967                                         rte_stats_bitrate_calc(bitrate_data,
 968                                                 idx_port);
 969                                 tics_datum = tics_current;
 970                         }
 971                 }
 972 #endif
 973 #ifdef RTE_LIBRTE_LATENCY_STATS
 974                 if (latencystats_enabled != 0 &&
 975                                 latencystats_lcore_id == rte_lcore_id())
 976                         rte_latencystats_update();
 977 #endif
 978
 979         } while (! fc->stopped);
 980 }
 981
 982 static int
 983 start_pkt_forward_on_core(void *fwd_arg)
 984 {
 985         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
 986                              cur_fwd_config.fwd_eng->packet_fwd);
 987         return 0;
 988 }
 989
 990 /*
 991  * Run the TXONLY packet forwarding engine to send a single burst of packets.
 992  * Used to start communication flows in network loopback test configurations.
 993  */
 994 static int
 995 run_one_txonly_burst_on_core(void *fwd_arg)
 996 {
 997         struct fwd_lcore *fwd_lc;
 998         struct fwd_lcore tmp_lcore;
 999
1000         fwd_lc = (struct fwd_lcore *) fwd_arg;
1001         tmp_lcore = *fwd_lc;
1002         tmp_lcore.stopped = 1;
1003         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1004         return 0;
1005 }
1006
1007 /*
1008  * Launch packet forwarding:
1009  *     - Setup per-port forwarding context.
1010  *     - launch logical cores with their forwarding configuration.
1011  */
1012 static void
1013 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1014 {
1015         port_fwd_begin_t port_fwd_begin;
1016         unsigned int i;
1017         unsigned int lc_id;
1018         int diag;
1019
1020         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1021         if (port_fwd_begin != NULL) {
1022                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1023                         (*port_fwd_begin)(fwd_ports_ids[i]);
1024         }
1025         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1026                 lc_id = fwd_lcores_cpuids[i];
1027                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1028                         fwd_lcores[i]->stopped = 0;
1029                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1030                                                      fwd_lcores[i], lc_id);
1031                         if (diag != 0)
1032                                 printf("launch lcore %u failed - diag=%d\n",
1033                                        lc_id, diag);
1034                 }
1035         }
1036 }
1037
1038 /*
1039  * Launch packet forwarding configuration.
1040  */
1041 void
1042 start_packet_forwarding(int with_tx_first)
1043 {
1044         port_fwd_begin_t port_fwd_begin;
1045         port_fwd_end_t  port_fwd_end;
1046         struct rte_port *port;
1047         unsigned int i;
1048         portid_t   pt_id;
1049         streamid_t sm_id;
1050
1051         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1052                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1053
1054         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1055                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1056
1057         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1058                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1059                 (!nb_rxq || !nb_txq))
1060                 rte_exit(EXIT_FAILURE,
1061                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1062                         cur_fwd_eng->fwd_mode_name);
1063
1064         if (all_ports_started() == 0) {
1065                 printf("Not all ports were started\n");
1066                 return;
1067         }
1068         if (test_done == 0) {
1069                 printf("Packet forwarding already started\n");
1070                 return;
1071         }
1072
1073         if (init_fwd_streams() < 0) {
1074                 printf("Fail from init_fwd_streams()\n");
1075                 return;
1076         }
1077
1078         if(dcb_test) {
1079                 for (i = 0; i < nb_fwd_ports; i++) {
1080                         pt_id = fwd_ports_ids[i];
1081                         port = &ports[pt_id];
1082                         if (!port->dcb_flag) {
1083                                 printf("In DCB mode, all forwarding ports must "
1084                                        "be configured in this mode.\n");
1085                                 return;
1086                         }
1087                 }
1088                 if (nb_fwd_lcores == 1) {
1089                         printf("In DCB mode,the nb forwarding cores "
1090                                "should be larger than 1.\n");
1091                         return;
1092                 }
1093         }
1094         test_done = 0;
1095
1096         if(!no_flush_rx)
1097                 flush_fwd_rx_queues();
1098
1099         fwd_config_setup();
1100         pkt_fwd_config_display(&cur_fwd_config);
1101         rxtx_config_display();
1102
1103         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1104                 pt_id = fwd_ports_ids[i];
1105                 port = &ports[pt_id];
1106                 rte_eth_stats_get(pt_id, &port->stats);
1107                 port->tx_dropped = 0;
1108
1109                 map_port_queue_stats_mapping_registers(pt_id, port);
1110         }
1111         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1112                 fwd_streams[sm_id]->rx_packets = 0;
1113                 fwd_streams[sm_id]->tx_packets = 0;
1114                 fwd_streams[sm_id]->fwd_dropped = 0;
1115                 fwd_streams[sm_id]->rx_bad_ip_csum = 0;
1116                 fwd_streams[sm_id]->rx_bad_l4_csum = 0;
1117
1118 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1119                 memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
1120                        sizeof(fwd_streams[sm_id]->rx_burst_stats));
1121                 memset(&fwd_streams[sm_id]->tx_burst_stats, 0,
1122                        sizeof(fwd_streams[sm_id]->tx_burst_stats));
1123 #endif
1124 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1125                 fwd_streams[sm_id]->core_cycles = 0;
1126 #endif
1127         }
1128         if (with_tx_first) {
1129                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1130                 if (port_fwd_begin != NULL) {
1131                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1132                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1133                 }
1134                 while (with_tx_first--) {
1135                         launch_packet_forwarding(
1136                                         run_one_txonly_burst_on_core);
1137                         rte_eal_mp_wait_lcore();
1138                 }
1139                 port_fwd_end = tx_only_engine.port_fwd_end;
1140                 if (port_fwd_end != NULL) {
1141                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1142                                 (*port_fwd_end)(fwd_ports_ids[i]);
1143                 }
1144         }
1145         launch_packet_forwarding(start_pkt_forward_on_core);
1146 }
1147
1148 void
1149 stop_packet_forwarding(void)
1150 {
1151         struct rte_eth_stats stats;
1152         struct rte_port *port;
1153         port_fwd_end_t  port_fwd_end;
1154         int i;
1155         portid_t   pt_id;
1156         streamid_t sm_id;
1157         lcoreid_t  lc_id;
1158         uint64_t total_recv;
1159         uint64_t total_xmit;
1160         uint64_t total_rx_dropped;
1161         uint64_t total_tx_dropped;
1162         uint64_t total_rx_nombuf;
1163         uint64_t tx_dropped;
1164         uint64_t rx_bad_ip_csum;
1165         uint64_t rx_bad_l4_csum;
1166 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1167         uint64_t fwd_cycles;
1168 #endif
1169         static const char *acc_stats_border = "+++++++++++++++";
1170
1171         if (test_done) {
1172                 printf("Packet forwarding not started\n");
1173                 return;
1174         }
1175         printf("Telling cores to stop...");
1176         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1177                 fwd_lcores[lc_id]->stopped = 1;
1178         printf("\nWaiting for lcores to finish...\n");
1179         rte_eal_mp_wait_lcore();
1180         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1181         if (port_fwd_end != NULL) {
1182                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1183                         pt_id = fwd_ports_ids[i];
1184                         (*port_fwd_end)(pt_id);
1185                 }
1186         }
1187 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1188         fwd_cycles = 0;
1189 #endif
1190         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1191                 if (cur_fwd_config.nb_fwd_streams >
1192                     cur_fwd_config.nb_fwd_ports) {
1193                         fwd_stream_stats_display(sm_id);
1194                         ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL;
1195                         ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL;
1196                 } else {
1197                         ports[fwd_streams[sm_id]->tx_port].tx_stream =
1198                                 fwd_streams[sm_id];
1199                         ports[fwd_streams[sm_id]->rx_port].rx_stream =
1200                                 fwd_streams[sm_id];
1201                 }
1202                 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped;
1203                 tx_dropped = (uint64_t) (tx_dropped +
1204                                          fwd_streams[sm_id]->fwd_dropped);
1205                 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped;
1206
1207                 rx_bad_ip_csum =
1208                         ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum;
1209                 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum +
1210                                          fwd_streams[sm_id]->rx_bad_ip_csum);
1211                 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum =
1212                                                         rx_bad_ip_csum;
1213
1214                 rx_bad_l4_csum =
1215                         ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum;
1216                 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum +
1217                                          fwd_streams[sm_id]->rx_bad_l4_csum);
1218                 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum =
1219                                                         rx_bad_l4_csum;
1220
1221 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1222                 fwd_cycles = (uint64_t) (fwd_cycles +
1223                                          fwd_streams[sm_id]->core_cycles);
1224 #endif
1225         }
1226         total_recv = 0;
1227         total_xmit = 0;
1228         total_rx_dropped = 0;
1229         total_tx_dropped = 0;
1230         total_rx_nombuf  = 0;
1231         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1232                 pt_id = fwd_ports_ids[i];
1233
1234                 port = &ports[pt_id];
1235                 rte_eth_stats_get(pt_id, &stats);
1236                 stats.ipackets -= port->stats.ipackets;
1237                 port->stats.ipackets = 0;
1238                 stats.opackets -= port->stats.opackets;
1239                 port->stats.opackets = 0;
1240                 stats.ibytes   -= port->stats.ibytes;
1241                 port->stats.ibytes = 0;
1242                 stats.obytes   -= port->stats.obytes;
1243                 port->stats.obytes = 0;
1244                 stats.imissed  -= port->stats.imissed;
1245                 port->stats.imissed = 0;
1246                 stats.oerrors  -= port->stats.oerrors;
1247                 port->stats.oerrors = 0;
1248                 stats.rx_nombuf -= port->stats.rx_nombuf;
1249                 port->stats.rx_nombuf = 0;
1250
1251                 total_recv += stats.ipackets;
1252                 total_xmit += stats.opackets;
1253                 total_rx_dropped += stats.imissed;
1254                 total_tx_dropped += port->tx_dropped;
1255                 total_rx_nombuf  += stats.rx_nombuf;
1256
1257                 fwd_port_stats_display(pt_id, &stats);
1258         }
1259         printf("\n  %s Accumulated forward statistics for all ports"
1260                "%s\n",
1261                acc_stats_border, acc_stats_border);
1262         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1263                "%-"PRIu64"\n"
1264                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1265                "%-"PRIu64"\n",
1266                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1267                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1268         if (total_rx_nombuf > 0)
1269                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1270         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1271                "%s\n",
1272                acc_stats_border, acc_stats_border);
1273 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1274         if (total_recv > 0)
1275                 printf("\n  CPU cycles/packet=%u (total cycles="
1276                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1277                        (unsigned int)(fwd_cycles / total_recv),
1278                        fwd_cycles, total_recv);
1279 #endif
1280         printf("\nDone.\n");
1281         test_done = 1;
1282 }
1283
1284 void
1285 dev_set_link_up(portid_t pid)
1286 {
1287         if (rte_eth_dev_set_link_up((uint8_t)pid) < 0)
1288                 printf("\nSet link up fail.\n");
1289 }
1290
1291 void
1292 dev_set_link_down(portid_t pid)
1293 {
1294         if (rte_eth_dev_set_link_down((uint8_t)pid) < 0)
1295                 printf("\nSet link down fail.\n");
1296 }
1297
1298 static int
1299 all_ports_started(void)
1300 {
1301         portid_t pi;
1302         struct rte_port *port;
1303
1304         RTE_ETH_FOREACH_DEV(pi) {
1305                 port = &ports[pi];
1306                 /* Check if there is a port which is not started */
1307                 if ((port->port_status != RTE_PORT_STARTED) &&
1308                         (port->slave_flag == 0))
1309                         return 0;
1310         }
1311
1312         /* No port is not started */
1313         return 1;
1314 }
1315
1316 int
1317 all_ports_stopped(void)
1318 {
1319         portid_t pi;
1320         struct rte_port *port;
1321
1322         RTE_ETH_FOREACH_DEV(pi) {
1323                 port = &ports[pi];
1324                 if ((port->port_status != RTE_PORT_STOPPED) &&
1325                         (port->slave_flag == 0))
1326                         return 0;
1327         }
1328
1329         return 1;
1330 }
1331
1332 int
1333 port_is_started(portid_t port_id)
1334 {
1335         if (port_id_is_invalid(port_id, ENABLED_WARN))
1336                 return 0;
1337
1338         if (ports[port_id].port_status != RTE_PORT_STARTED)
1339                 return 0;
1340
1341         return 1;
1342 }
1343
1344 static int
1345 port_is_closed(portid_t port_id)
1346 {
1347         if (port_id_is_invalid(port_id, ENABLED_WARN))
1348                 return 0;
1349
1350         if (ports[port_id].port_status != RTE_PORT_CLOSED)
1351                 return 0;
1352
1353         return 1;
1354 }
1355
1356 int
1357 start_port(portid_t pid)
1358 {
1359         int diag, need_check_link_status = -1;
1360         portid_t pi;
1361         queueid_t qi;
1362         struct rte_port *port;
1363         struct ether_addr mac_addr;
1364         enum rte_eth_event_type event_type;
1365
1366         if (port_id_is_invalid(pid, ENABLED_WARN))
1367                 return 0;
1368
1369         if(dcb_config)
1370                 dcb_test = 1;
1371         RTE_ETH_FOREACH_DEV(pi) {
1372                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1373                         continue;
1374
1375                 need_check_link_status = 0;
1376                 port = &ports[pi];
1377                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1378                                                  RTE_PORT_HANDLING) == 0) {
1379                         printf("Port %d is now not stopped\n", pi);
1380                         continue;
1381                 }
1382
1383                 if (port->need_reconfig > 0) {
1384                         port->need_reconfig = 0;
1385
1386                         printf("Configuring Port %d (socket %u)\n", pi,
1387                                         port->socket_id);
1388                         /* configure port */
1389                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1390                                                 &(port->dev_conf));
1391                         if (diag != 0) {
1392                                 if (rte_atomic16_cmpset(&(port->port_status),
1393                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1394                                         printf("Port %d can not be set back "
1395                                                         "to stopped\n", pi);
1396                                 printf("Fail to configure port %d\n", pi);
1397                                 /* try to reconfigure port next time */
1398                                 port->need_reconfig = 1;
1399                                 return -1;
1400                         }
1401                 }
1402                 if (port->need_reconfig_queues > 0) {
1403                         port->need_reconfig_queues = 0;
1404                         /* setup tx queues */
1405                         for (qi = 0; qi < nb_txq; qi++) {
1406                                 if ((numa_support) &&
1407                                         (txring_numa[pi] != NUMA_NO_CONFIG))
1408                                         diag = rte_eth_tx_queue_setup(pi, qi,
1409                                                 nb_txd,txring_numa[pi],
1410                                                 &(port->tx_conf));
1411                                 else
1412                                         diag = rte_eth_tx_queue_setup(pi, qi,
1413                                                 nb_txd,port->socket_id,
1414                                                 &(port->tx_conf));
1415
1416                                 if (diag == 0)
1417                                         continue;
1418
1419                                 /* Fail to setup tx queue, return */
1420                                 if (rte_atomic16_cmpset(&(port->port_status),
1421                                                         RTE_PORT_HANDLING,
1422                                                         RTE_PORT_STOPPED) == 0)
1423                                         printf("Port %d can not be set back "
1424                                                         "to stopped\n", pi);
1425                                 printf("Fail to configure port %d tx queues\n", pi);
1426                                 /* try to reconfigure queues next time */
1427                                 port->need_reconfig_queues = 1;
1428                                 return -1;
1429                         }
1430                         /* setup rx queues */
1431                         for (qi = 0; qi < nb_rxq; qi++) {
1432                                 if ((numa_support) &&
1433                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
1434                                         struct rte_mempool * mp =
1435                                                 mbuf_pool_find(rxring_numa[pi]);
1436                                         if (mp == NULL) {
1437                                                 printf("Failed to setup RX queue:"
1438                                                         "No mempool allocation"
1439                                                         " on the socket %d\n",
1440                                                         rxring_numa[pi]);
1441                                                 return -1;
1442                                         }
1443
1444                                         diag = rte_eth_rx_queue_setup(pi, qi,
1445                                              nb_rxd,rxring_numa[pi],
1446                                              &(port->rx_conf),mp);
1447                                 } else {
1448                                         struct rte_mempool *mp =
1449                                                 mbuf_pool_find(port->socket_id);
1450                                         if (mp == NULL) {
1451                                                 printf("Failed to setup RX queue:"
1452                                                         "No mempool allocation"
1453                                                         " on the socket %d\n",
1454                                                         port->socket_id);
1455                                                 return -1;
1456                                         }
1457                                         diag = rte_eth_rx_queue_setup(pi, qi,
1458                                              nb_rxd,port->socket_id,
1459                                              &(port->rx_conf), mp);
1460                                 }
1461                                 if (diag == 0)
1462                                         continue;
1463
1464                                 /* Fail to setup rx queue, return */
1465                                 if (rte_atomic16_cmpset(&(port->port_status),
1466                                                         RTE_PORT_HANDLING,
1467                                                         RTE_PORT_STOPPED) == 0)
1468                                         printf("Port %d can not be set back "
1469                                                         "to stopped\n", pi);
1470                                 printf("Fail to configure port %d rx queues\n", pi);
1471                                 /* try to reconfigure queues next time */
1472                                 port->need_reconfig_queues = 1;
1473                                 return -1;
1474                         }
1475                 }
1476
1477                 for (event_type = RTE_ETH_EVENT_UNKNOWN;
1478                      event_type < RTE_ETH_EVENT_MAX;
1479                      event_type++) {
1480                         diag = rte_eth_dev_callback_register(pi,
1481                                                         event_type,
1482                                                         eth_event_callback,
1483                                                         NULL);
1484                         if (diag) {
1485                                 printf("Failed to setup even callback for event %d\n",
1486                                         event_type);
1487                                 return -1;
1488                         }
1489                 }
1490
1491                 /* start port */
1492                 if (rte_eth_dev_start(pi) < 0) {
1493                         printf("Fail to start port %d\n", pi);
1494
1495                         /* Fail to setup rx queue, return */
1496                         if (rte_atomic16_cmpset(&(port->port_status),
1497                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1498                                 printf("Port %d can not be set back to "
1499                                                         "stopped\n", pi);
1500                         continue;
1501                 }
1502
1503                 if (rte_atomic16_cmpset(&(port->port_status),
1504                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
1505                         printf("Port %d can not be set into started\n", pi);
1506
1507                 rte_eth_macaddr_get(pi, &mac_addr);
1508                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
1509                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
1510                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
1511                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
1512
1513                 /* at least one port started, need checking link status */
1514                 need_check_link_status = 1;
1515         }
1516
1517         if (need_check_link_status == 1 && !no_link_check)
1518                 check_all_ports_link_status(RTE_PORT_ALL);
1519         else if (need_check_link_status == 0)
1520                 printf("Please stop the ports first\n");
1521
1522         printf("Done\n");
1523         return 0;
1524 }
1525
1526 void
1527 stop_port(portid_t pid)
1528 {
1529         portid_t pi;
1530         struct rte_port *port;
1531         int need_check_link_status = 0;
1532
1533         if (dcb_test) {
1534                 dcb_test = 0;
1535                 dcb_config = 0;
1536         }
1537
1538         if (port_id_is_invalid(pid, ENABLED_WARN))
1539                 return;
1540
1541         printf("Stopping ports...\n");
1542
1543         RTE_ETH_FOREACH_DEV(pi) {
1544                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1545                         continue;
1546
1547                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1548                         printf("Please remove port %d from forwarding configuration.\n", pi);
1549                         continue;
1550                 }
1551
1552                 if (port_is_bonding_slave(pi)) {
1553                         printf("Please remove port %d from bonded device.\n", pi);
1554                         continue;
1555                 }
1556
1557                 port = &ports[pi];
1558                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
1559                                                 RTE_PORT_HANDLING) == 0)
1560                         continue;
1561
1562                 rte_eth_dev_stop(pi);
1563
1564                 if (rte_atomic16_cmpset(&(port->port_status),
1565                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1566                         printf("Port %d can not be set into stopped\n", pi);
1567                 need_check_link_status = 1;
1568         }
1569         if (need_check_link_status && !no_link_check)
1570                 check_all_ports_link_status(RTE_PORT_ALL);
1571
1572         printf("Done\n");
1573 }
1574
1575 void
1576 close_port(portid_t pid)
1577 {
1578         portid_t pi;
1579         struct rte_port *port;
1580
1581         if (port_id_is_invalid(pid, ENABLED_WARN))
1582                 return;
1583
1584         printf("Closing ports...\n");
1585
1586         RTE_ETH_FOREACH_DEV(pi) {
1587                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1588                         continue;
1589
1590                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1591                         printf("Please remove port %d from forwarding configuration.\n", pi);
1592                         continue;
1593                 }
1594
1595                 if (port_is_bonding_slave(pi)) {
1596                         printf("Please remove port %d from bonded device.\n", pi);
1597                         continue;
1598                 }
1599
1600                 port = &ports[pi];
1601                 if (rte_atomic16_cmpset(&(port->port_status),
1602                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
1603                         printf("Port %d is already closed\n", pi);
1604                         continue;
1605                 }
1606
1607                 if (rte_atomic16_cmpset(&(port->port_status),
1608                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
1609                         printf("Port %d is now not stopped\n", pi);
1610                         continue;
1611                 }
1612
1613                 if (port->flow_list)
1614                         port_flow_flush(pi);
1615                 rte_eth_dev_close(pi);
1616
1617                 if (rte_atomic16_cmpset(&(port->port_status),
1618                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
1619                         printf("Port %d cannot be set to closed\n", pi);
1620         }
1621
1622         printf("Done\n");
1623 }
1624
1625 void
1626 attach_port(char *identifier)
1627 {
1628         portid_t pi = 0;
1629         unsigned int socket_id;
1630
1631         printf("Attaching a new port...\n");
1632
1633         if (identifier == NULL) {
1634                 printf("Invalid parameters are specified\n");
1635                 return;
1636         }
1637
1638         if (rte_eth_dev_attach(identifier, &pi))
1639                 return;
1640
1641         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
1642         /* if socket_id is invalid, set to 0 */
1643         if (check_socket_id(socket_id) < 0)
1644                 socket_id = 0;
1645         reconfig(pi, socket_id);
1646         rte_eth_promiscuous_enable(pi);
1647
1648         nb_ports = rte_eth_dev_count();
1649
1650         ports[pi].port_status = RTE_PORT_STOPPED;
1651
1652         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
1653         printf("Done\n");
1654 }
1655
1656 void
1657 detach_port(uint8_t port_id)
1658 {
1659         char name[RTE_ETH_NAME_MAX_LEN];
1660
1661         printf("Detaching a port...\n");
1662
1663         if (!port_is_closed(port_id)) {
1664                 printf("Please close port first\n");
1665                 return;
1666         }
1667
1668         if (ports[port_id].flow_list)
1669                 port_flow_flush(port_id);
1670
1671         if (rte_eth_dev_detach(port_id, name))
1672                 return;
1673
1674         nb_ports = rte_eth_dev_count();
1675
1676         printf("Port '%s' is detached. Now total ports is %d\n",
1677                         name, nb_ports);
1678         printf("Done\n");
1679         return;
1680 }
1681
1682 void
1683 pmd_test_exit(void)
1684 {
1685         portid_t pt_id;
1686
1687         if (test_done == 0)
1688                 stop_packet_forwarding();
1689
1690         if (ports != NULL) {
1691                 no_link_check = 1;
1692                 RTE_ETH_FOREACH_DEV(pt_id) {
1693                         printf("\nShutting down port %d...\n", pt_id);
1694                         fflush(stdout);
1695                         stop_port(pt_id);
1696                         close_port(pt_id);
1697                 }
1698         }
1699         printf("\nBye...\n");
1700 }
1701
1702 typedef void (*cmd_func_t)(void);
1703 struct pmd_test_command {
1704         const char *cmd_name;
1705         cmd_func_t cmd_func;
1706 };
1707
1708 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
1709
1710 /* Check the link status of all ports in up to 9s, and print them finally */
1711 static void
1712 check_all_ports_link_status(uint32_t port_mask)
1713 {
1714 #define CHECK_INTERVAL 100 /* 100ms */
1715 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
1716         uint8_t portid, count, all_ports_up, print_flag = 0;
1717         struct rte_eth_link link;
1718
1719         printf("Checking link statuses...\n");
1720         fflush(stdout);
1721         for (count = 0; count <= MAX_CHECK_TIME; count++) {
1722                 all_ports_up = 1;
1723                 RTE_ETH_FOREACH_DEV(portid) {
1724                         if ((port_mask & (1 << portid)) == 0)
1725                                 continue;
1726                         memset(&link, 0, sizeof(link));
1727                         rte_eth_link_get_nowait(portid, &link);
1728                         /* print link status if flag set */
1729                         if (print_flag == 1) {
1730                                 if (link.link_status)
1731                                         printf("Port %d Link Up - speed %u "
1732                                                 "Mbps - %s\n", (uint8_t)portid,
1733                                                 (unsigned)link.link_speed,
1734                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
1735                                         ("full-duplex") : ("half-duplex\n"));
1736                                 else
1737                                         printf("Port %d Link Down\n",
1738                                                 (uint8_t)portid);
1739                                 continue;
1740                         }
1741                         /* clear all_ports_up flag if any link down */
1742                         if (link.link_status == ETH_LINK_DOWN) {
1743                                 all_ports_up = 0;
1744                                 break;
1745                         }
1746                 }
1747                 /* after finally printing all link status, get out */
1748                 if (print_flag == 1)
1749                         break;
1750
1751                 if (all_ports_up == 0) {
1752                         fflush(stdout);
1753                         rte_delay_ms(CHECK_INTERVAL);
1754                 }
1755
1756                 /* set the print_flag if all ports up or timeout */
1757                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
1758                         print_flag = 1;
1759                 }
1760
1761                 if (lsc_interrupt)
1762                         break;
1763         }
1764 }
1765
1766 static void
1767 rmv_event_callback(void *arg)
1768 {
1769         struct rte_eth_dev *dev;
1770         struct rte_devargs *da;
1771         char name[32] = "";
1772         uint8_t port_id = (intptr_t)arg;
1773
1774         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1775         dev = &rte_eth_devices[port_id];
1776         da = dev->device->devargs;
1777
1778         stop_port(port_id);
1779         close_port(port_id);
1780         if (da->type == RTE_DEVTYPE_VIRTUAL)
1781                 snprintf(name, sizeof(name), "%s", da->virt.drv_name);
1782         else if (da->type == RTE_DEVTYPE_WHITELISTED_PCI)
1783                 rte_pci_device_name(&da->pci.addr, name, sizeof(name));
1784         printf("removing device %s\n", name);
1785         rte_eal_dev_detach(name);
1786         dev->state = RTE_ETH_DEV_UNUSED;
1787 }
1788
1789 /* This function is used by the interrupt thread */
1790 static void
1791 eth_event_callback(uint8_t port_id, enum rte_eth_event_type type, void *param)
1792 {
1793         static const char * const event_desc[] = {
1794                 [RTE_ETH_EVENT_UNKNOWN] = "Unknown",
1795                 [RTE_ETH_EVENT_INTR_LSC] = "LSC",
1796                 [RTE_ETH_EVENT_QUEUE_STATE] = "Queue state",
1797                 [RTE_ETH_EVENT_INTR_RESET] = "Interrupt reset",
1798                 [RTE_ETH_EVENT_VF_MBOX] = "VF Mbox",
1799                 [RTE_ETH_EVENT_MACSEC] = "MACsec",
1800                 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
1801                 [RTE_ETH_EVENT_MAX] = NULL,
1802         };
1803
1804         RTE_SET_USED(param);
1805
1806         if (type >= RTE_ETH_EVENT_MAX) {
1807                 fprintf(stderr, "\nPort %" PRIu8 ": %s called upon invalid event %d\n",
1808                         port_id, __func__, type);
1809                 fflush(stderr);
1810         } else {
1811                 printf("\nPort %" PRIu8 ": %s event\n", port_id,
1812                         event_desc[type]);
1813                 fflush(stdout);
1814         }
1815
1816         switch (type) {
1817         case RTE_ETH_EVENT_INTR_RMV:
1818                 if (rte_eal_alarm_set(100000,
1819                                 rmv_event_callback, (void *)(intptr_t)port_id))
1820                         fprintf(stderr, "Could not set up deferred device removal\n");
1821                 break;
1822         default:
1823                 break;
1824         }
1825 }
1826
1827 static int
1828 set_tx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port)
1829 {
1830         uint16_t i;
1831         int diag;
1832         uint8_t mapping_found = 0;
1833
1834         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
1835                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
1836                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
1837                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
1838                                         tx_queue_stats_mappings[i].queue_id,
1839                                         tx_queue_stats_mappings[i].stats_counter_id);
1840                         if (diag != 0)
1841                                 return diag;
1842                         mapping_found = 1;
1843                 }
1844         }
1845         if (mapping_found)
1846                 port->tx_queue_stats_mapping_enabled = 1;
1847         return 0;
1848 }
1849
1850 static int
1851 set_rx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port)
1852 {
1853         uint16_t i;
1854         int diag;
1855         uint8_t mapping_found = 0;
1856
1857         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
1858                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
1859                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
1860                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
1861                                         rx_queue_stats_mappings[i].queue_id,
1862                                         rx_queue_stats_mappings[i].stats_counter_id);
1863                         if (diag != 0)
1864                                 return diag;
1865                         mapping_found = 1;
1866                 }
1867         }
1868         if (mapping_found)
1869                 port->rx_queue_stats_mapping_enabled = 1;
1870         return 0;
1871 }
1872
1873 static void
1874 map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port)
1875 {
1876         int diag = 0;
1877
1878         diag = set_tx_queue_stats_mapping_registers(pi, port);
1879         if (diag != 0) {
1880                 if (diag == -ENOTSUP) {
1881                         port->tx_queue_stats_mapping_enabled = 0;
1882                         printf("TX queue stats mapping not supported port id=%d\n", pi);
1883                 }
1884                 else
1885                         rte_exit(EXIT_FAILURE,
1886                                         "set_tx_queue_stats_mapping_registers "
1887                                         "failed for port id=%d diag=%d\n",
1888                                         pi, diag);
1889         }
1890
1891         diag = set_rx_queue_stats_mapping_registers(pi, port);
1892         if (diag != 0) {
1893                 if (diag == -ENOTSUP) {
1894                         port->rx_queue_stats_mapping_enabled = 0;
1895                         printf("RX queue stats mapping not supported port id=%d\n", pi);
1896                 }
1897                 else
1898                         rte_exit(EXIT_FAILURE,
1899                                         "set_rx_queue_stats_mapping_registers "
1900                                         "failed for port id=%d diag=%d\n",
1901                                         pi, diag);
1902         }
1903 }
1904
1905 static void
1906 rxtx_port_config(struct rte_port *port)
1907 {
1908         port->rx_conf = port->dev_info.default_rxconf;
1909         port->tx_conf = port->dev_info.default_txconf;
1910
1911         /* Check if any RX/TX parameters have been passed */
1912         if (rx_pthresh != RTE_PMD_PARAM_UNSET)
1913                 port->rx_conf.rx_thresh.pthresh = rx_pthresh;
1914
1915         if (rx_hthresh != RTE_PMD_PARAM_UNSET)
1916                 port->rx_conf.rx_thresh.hthresh = rx_hthresh;
1917
1918         if (rx_wthresh != RTE_PMD_PARAM_UNSET)
1919                 port->rx_conf.rx_thresh.wthresh = rx_wthresh;
1920
1921         if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
1922                 port->rx_conf.rx_free_thresh = rx_free_thresh;
1923
1924         if (rx_drop_en != RTE_PMD_PARAM_UNSET)
1925                 port->rx_conf.rx_drop_en = rx_drop_en;
1926
1927         if (tx_pthresh != RTE_PMD_PARAM_UNSET)
1928                 port->tx_conf.tx_thresh.pthresh = tx_pthresh;
1929
1930         if (tx_hthresh != RTE_PMD_PARAM_UNSET)
1931                 port->tx_conf.tx_thresh.hthresh = tx_hthresh;
1932
1933         if (tx_wthresh != RTE_PMD_PARAM_UNSET)
1934                 port->tx_conf.tx_thresh.wthresh = tx_wthresh;
1935
1936         if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
1937                 port->tx_conf.tx_rs_thresh = tx_rs_thresh;
1938
1939         if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
1940                 port->tx_conf.tx_free_thresh = tx_free_thresh;
1941
1942         if (txq_flags != RTE_PMD_PARAM_UNSET)
1943                 port->tx_conf.txq_flags = txq_flags;
1944 }
1945
1946 void
1947 init_port_config(void)
1948 {
1949         portid_t pid;
1950         struct rte_port *port;
1951
1952         RTE_ETH_FOREACH_DEV(pid) {
1953                 port = &ports[pid];
1954                 port->dev_conf.rxmode = rx_mode;
1955                 port->dev_conf.fdir_conf = fdir_conf;
1956                 if (nb_rxq > 1) {
1957                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1958                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = rss_hf;
1959                 } else {
1960                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1961                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
1962                 }
1963
1964                 if (port->dcb_flag == 0) {
1965                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
1966                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
1967                         else
1968                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
1969                 }
1970
1971                 rxtx_port_config(port);
1972
1973                 rte_eth_macaddr_get(pid, &port->eth_addr);
1974
1975                 map_port_queue_stats_mapping_registers(pid, port);
1976 #ifdef RTE_NIC_BYPASS
1977                 rte_eth_dev_bypass_init(pid);
1978 #endif
1979
1980                 if (lsc_interrupt &&
1981                     (rte_eth_devices[pid].data->dev_flags &
1982                      RTE_ETH_DEV_INTR_LSC))
1983                         port->dev_conf.intr_conf.lsc = 1;
1984                 if (rmv_interrupt &&
1985                     (rte_eth_devices[pid].data->dev_flags &
1986                      RTE_ETH_DEV_INTR_RMV))
1987                         port->dev_conf.intr_conf.rmv = 1;
1988         }
1989 }
1990
1991 void set_port_slave_flag(portid_t slave_pid)
1992 {
1993         struct rte_port *port;
1994
1995         port = &ports[slave_pid];
1996         port->slave_flag = 1;
1997 }
1998
1999 void clear_port_slave_flag(portid_t slave_pid)
2000 {
2001         struct rte_port *port;
2002
2003         port = &ports[slave_pid];
2004         port->slave_flag = 0;
2005 }
2006
2007 uint8_t port_is_bonding_slave(portid_t slave_pid)
2008 {
2009         struct rte_port *port;
2010
2011         port = &ports[slave_pid];
2012         return port->slave_flag;
2013 }
2014
2015 const uint16_t vlan_tags[] = {
2016                 0,  1,  2,  3,  4,  5,  6,  7,
2017                 8,  9, 10, 11,  12, 13, 14, 15,
2018                 16, 17, 18, 19, 20, 21, 22, 23,
2019                 24, 25, 26, 27, 28, 29, 30, 31
2020 };
2021
2022 static  int
2023 get_eth_dcb_conf(struct rte_eth_conf *eth_conf,
2024                  enum dcb_mode_enable dcb_mode,
2025                  enum rte_eth_nb_tcs num_tcs,
2026                  uint8_t pfc_en)
2027 {
2028         uint8_t i;
2029
2030         /*
2031          * Builds up the correct configuration for dcb+vt based on the vlan tags array
2032          * given above, and the number of traffic classes available for use.
2033          */
2034         if (dcb_mode == DCB_VT_ENABLED) {
2035                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
2036                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
2037                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2038                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
2039
2040                 /* VMDQ+DCB RX and TX configurations */
2041                 vmdq_rx_conf->enable_default_pool = 0;
2042                 vmdq_rx_conf->default_pool = 0;
2043                 vmdq_rx_conf->nb_queue_pools =
2044                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2045                 vmdq_tx_conf->nb_queue_pools =
2046                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2047
2048                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
2049                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
2050                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
2051                         vmdq_rx_conf->pool_map[i].pools =
2052                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
2053                 }
2054                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2055                         vmdq_rx_conf->dcb_tc[i] = i;
2056                         vmdq_tx_conf->dcb_tc[i] = i;
2057                 }
2058
2059                 /* set DCB mode of RX and TX of multiple queues */
2060                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
2061                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
2062         } else {
2063                 struct rte_eth_dcb_rx_conf *rx_conf =
2064                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
2065                 struct rte_eth_dcb_tx_conf *tx_conf =
2066                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
2067
2068                 rx_conf->nb_tcs = num_tcs;
2069                 tx_conf->nb_tcs = num_tcs;
2070
2071                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2072                         rx_conf->dcb_tc[i] = i % num_tcs;
2073                         tx_conf->dcb_tc[i] = i % num_tcs;
2074                 }
2075                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
2076                 eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf;
2077                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
2078         }
2079
2080         if (pfc_en)
2081                 eth_conf->dcb_capability_en =
2082                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
2083         else
2084                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
2085
2086         return 0;
2087 }
2088
2089 int
2090 init_port_dcb_config(portid_t pid,
2091                      enum dcb_mode_enable dcb_mode,
2092                      enum rte_eth_nb_tcs num_tcs,
2093                      uint8_t pfc_en)
2094 {
2095         struct rte_eth_conf port_conf;
2096         struct rte_port *rte_port;
2097         int retval;
2098         uint16_t i;
2099
2100         rte_port = &ports[pid];
2101
2102         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
2103         /* Enter DCB configuration status */
2104         dcb_config = 1;
2105
2106         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
2107         retval = get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en);
2108         if (retval < 0)
2109                 return retval;
2110         port_conf.rxmode.hw_vlan_filter = 1;
2111
2112         /**
2113          * Write the configuration into the device.
2114          * Set the numbers of RX & TX queues to 0, so
2115          * the RX & TX queues will not be setup.
2116          */
2117         (void)rte_eth_dev_configure(pid, 0, 0, &port_conf);
2118
2119         rte_eth_dev_info_get(pid, &rte_port->dev_info);
2120
2121         /* If dev_info.vmdq_pool_base is greater than 0,
2122          * the queue id of vmdq pools is started after pf queues.
2123          */
2124         if (dcb_mode == DCB_VT_ENABLED &&
2125             rte_port->dev_info.vmdq_pool_base > 0) {
2126                 printf("VMDQ_DCB multi-queue mode is nonsensical"
2127                         " for port %d.", pid);
2128                 return -1;
2129         }
2130
2131         /* Assume the ports in testpmd have the same dcb capability
2132          * and has the same number of rxq and txq in dcb mode
2133          */
2134         if (dcb_mode == DCB_VT_ENABLED) {
2135                 if (rte_port->dev_info.max_vfs > 0) {
2136                         nb_rxq = rte_port->dev_info.nb_rx_queues;
2137                         nb_txq = rte_port->dev_info.nb_tx_queues;
2138                 } else {
2139                         nb_rxq = rte_port->dev_info.max_rx_queues;
2140                         nb_txq = rte_port->dev_info.max_tx_queues;
2141                 }
2142         } else {
2143                 /*if vt is disabled, use all pf queues */
2144                 if (rte_port->dev_info.vmdq_pool_base == 0) {
2145                         nb_rxq = rte_port->dev_info.max_rx_queues;
2146                         nb_txq = rte_port->dev_info.max_tx_queues;
2147                 } else {
2148                         nb_rxq = (queueid_t)num_tcs;
2149                         nb_txq = (queueid_t)num_tcs;
2150
2151                 }
2152         }
2153         rx_free_thresh = 64;
2154
2155         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
2156
2157         rxtx_port_config(rte_port);
2158         /* VLAN filter */
2159         rte_port->dev_conf.rxmode.hw_vlan_filter = 1;
2160         for (i = 0; i < RTE_DIM(vlan_tags); i++)
2161                 rx_vft_set(pid, vlan_tags[i], 1);
2162
2163         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
2164         map_port_queue_stats_mapping_registers(pid, rte_port);
2165
2166         rte_port->dcb_flag = 1;
2167
2168         return 0;
2169 }
2170
2171 static void
2172 init_port(void)
2173 {
2174         /* Configuration of Ethernet ports. */
2175         ports = rte_zmalloc("testpmd: ports",
2176                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
2177                             RTE_CACHE_LINE_SIZE);
2178         if (ports == NULL) {
2179                 rte_exit(EXIT_FAILURE,
2180                                 "rte_zmalloc(%d struct rte_port) failed\n",
2181                                 RTE_MAX_ETHPORTS);
2182         }
2183 }
2184
2185 static void
2186 force_quit(void)
2187 {
2188         pmd_test_exit();
2189         prompt_exit();
2190 }
2191
2192 static void
2193 signal_handler(int signum)
2194 {
2195         if (signum == SIGINT || signum == SIGTERM) {
2196                 printf("\nSignal %d received, preparing to exit...\n",
2197                                 signum);
2198 #ifdef RTE_LIBRTE_PDUMP
2199                 /* uninitialize packet capture framework */
2200                 rte_pdump_uninit();
2201 #endif
2202 #ifdef RTE_LIBRTE_LATENCY_STATS
2203                 rte_latencystats_uninit();
2204 #endif
2205                 force_quit();
2206                 /* exit with the expected status */
2207                 signal(signum, SIG_DFL);
2208                 kill(getpid(), signum);
2209         }
2210 }
2211
2212 int
2213 main(int argc, char** argv)
2214 {
2215         int  diag;
2216         uint8_t port_id;
2217
2218         signal(SIGINT, signal_handler);
2219         signal(SIGTERM, signal_handler);
2220
2221         diag = rte_eal_init(argc, argv);
2222         if (diag < 0)
2223                 rte_panic("Cannot init EAL\n");
2224
2225 #ifdef RTE_LIBRTE_PDUMP
2226         /* initialize packet capture framework */
2227         rte_pdump_init(NULL);
2228 #endif
2229
2230         nb_ports = (portid_t) rte_eth_dev_count();
2231         if (nb_ports == 0)
2232                 RTE_LOG(WARNING, EAL, "No probed ethernet devices\n");
2233
2234         /* allocate port structures, and init them */
2235         init_port();
2236
2237         set_def_fwd_config();
2238         if (nb_lcores == 0)
2239                 rte_panic("Empty set of forwarding logical cores - check the "
2240                           "core mask supplied in the command parameters\n");
2241
2242         /* Bitrate/latency stats disabled by default */
2243 #ifdef RTE_LIBRTE_BITRATE
2244         bitrate_enabled = 0;
2245 #endif
2246 #ifdef RTE_LIBRTE_LATENCY_STATS
2247         latencystats_enabled = 0;
2248 #endif
2249
2250         argc -= diag;
2251         argv += diag;
2252         if (argc > 1)
2253                 launch_args_parse(argc, argv);
2254
2255         if (!nb_rxq && !nb_txq)
2256                 printf("Warning: Either rx or tx queues should be non-zero\n");
2257
2258         if (nb_rxq > 1 && nb_rxq > nb_txq)
2259                 printf("Warning: nb_rxq=%d enables RSS configuration, "
2260                        "but nb_txq=%d will prevent to fully test it.\n",
2261                        nb_rxq, nb_txq);
2262
2263         init_config();
2264         if (start_port(RTE_PORT_ALL) != 0)
2265                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
2266
2267         /* set all ports to promiscuous mode by default */
2268         RTE_ETH_FOREACH_DEV(port_id)
2269                 rte_eth_promiscuous_enable(port_id);
2270
2271         /* Init metrics library */
2272         rte_metrics_init(rte_socket_id());
2273
2274 #ifdef RTE_LIBRTE_LATENCY_STATS
2275         if (latencystats_enabled != 0) {
2276                 int ret = rte_latencystats_init(1, NULL);
2277                 if (ret)
2278                         printf("Warning: latencystats init()"
2279                                 " returned error %d\n", ret);
2280                 printf("Latencystats running on lcore %d\n",
2281                         latencystats_lcore_id);
2282         }
2283 #endif
2284
2285         /* Setup bitrate stats */
2286 #ifdef RTE_LIBRTE_BITRATE
2287         if (bitrate_enabled != 0) {
2288                 bitrate_data = rte_stats_bitrate_create();
2289                 if (bitrate_data == NULL)
2290                         rte_exit(EXIT_FAILURE,
2291                                 "Could not allocate bitrate data.\n");
2292                 rte_stats_bitrate_reg(bitrate_data);
2293         }
2294 #endif
2295
2296 #ifdef RTE_LIBRTE_CMDLINE
2297         if (strlen(cmdline_filename) != 0)
2298                 cmdline_read_from_file(cmdline_filename);
2299
2300         if (interactive == 1) {
2301                 if (auto_start) {
2302                         printf("Start automatic packet forwarding\n");
2303                         start_packet_forwarding(0);
2304                 }
2305                 prompt();
2306                 pmd_test_exit();
2307         } else
2308 #endif
2309         {
2310                 char c;
2311                 int rc;
2312
2313                 printf("No commandline core given, start packet forwarding\n");
2314                 start_packet_forwarding(0);
2315                 printf("Press enter to exit\n");
2316                 rc = read(0, &c, 1);
2317                 pmd_test_exit();
2318                 if (rc < 0)
2319                         return 1;
2320         }
2321
2322         return 0;
2323 }