app/test-pmd/testpmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2013 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdarg.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <signal.h>
  38 #include <string.h>
  39 #include <time.h>
  40 #include <fcntl.h>
  41 #include <sys/types.h>
  42 #include <errno.h>
  43
  44 #include <sys/queue.h>
  45 #include <sys/stat.h>
  46
  47 #include <stdint.h>
  48 #include <unistd.h>
  49 #include <inttypes.h>
  50
  51 #include <rte_common.h>
  52 #include <rte_byteorder.h>
  53 #include <rte_log.h>
  54 #include <rte_debug.h>
  55 #include <rte_cycles.h>
  56 #include <rte_memory.h>
  57 #include <rte_memcpy.h>
  58 #include <rte_memzone.h>
  59 #include <rte_launch.h>
  60 #include <rte_tailq.h>
  61 #include <rte_eal.h>
  62 #include <rte_per_lcore.h>
  63 #include <rte_lcore.h>
  64 #include <rte_atomic.h>
  65 #include <rte_branch_prediction.h>
  66 #include <rte_ring.h>
  67 #include <rte_mempool.h>
  68 #include <rte_malloc.h>
  69 #include <rte_mbuf.h>
  70 #include <rte_interrupts.h>
  71 #include <rte_pci.h>
  72 #include <rte_ether.h>
  73 #include <rte_ethdev.h>
  74 #include <rte_string_fns.h>
  75
  76 #include "testpmd.h"
  77
  78 uint16_t verbose_level = 0; /**< Silent by default. */
  79
  80 /* use master core for command line ? */
  81 uint8_t interactive = 0;
  82
  83 /*
  84  * NUMA support configuration.
  85  * When set, the NUMA support attempts to dispatch the allocation of the
  86  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  87  * probed ports among the CPU sockets 0 and 1.
  88  * Otherwise, all memory is allocated from CPU socket 0.
  89  */
  90 uint8_t numa_support = 0; /**< No numa support by default */
  91
  92 /*
  93  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
  94  * not configured.
  95  */
  96 uint8_t socket_num = UMA_NO_CONFIG;
  97
  98 /*
  99  * Record the Ethernet address of peer target ports to which packets are
 100  * forwarded.
 101  * Must be instanciated with the ethernet addresses of peer traffic generator
 102  * ports.
 103  */
 104 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 105 portid_t nb_peer_eth_addrs = 0;
 106
 107 /*
 108  * Probed Target Environment.
 109  */
 110 struct rte_port *ports;        /**< For all probed ethernet ports. */
 111 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 112 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 113 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 114
 115 /*
 116  * Test Forwarding Configuration.
 117  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 118  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 119  */
 120 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 121 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 122 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 123 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 124
 125 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 126 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 127
 128 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 129 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 130
 131 /*
 132  * Forwarding engines.
 133  */
 134 struct fwd_engine * fwd_engines[] = {
 135         &io_fwd_engine,
 136         &mac_fwd_engine,
 137         &rx_only_engine,
 138         &tx_only_engine,
 139         &csum_fwd_engine,
 140 #ifdef RTE_LIBRTE_IEEE1588
 141         &ieee1588_fwd_engine,
 142 #endif
 143         NULL,
 144 };
 145
 146 struct fwd_config cur_fwd_config;
 147 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 148
 149 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 150 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 151                                       * specified on command-line. */
 152
 153 /*
 154  * Configuration of packet segments used by the "txonly" processing engine.
 155  */
 156 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 157 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 158         TXONLY_DEF_PACKET_LEN,
 159 };
 160 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 161
 162 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 163 uint16_t mb_mempool_cache = DEF_PKT_BURST; /**< Size of mbuf mempool cache. */
 164
 165 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 166 uint8_t dcb_config = 0;
 167
 168 /* Whether the dcb is in testing status */
 169 uint8_t dcb_test = 0;
 170
 171 /* DCB on and VT on mapping is default */
 172 enum dcb_queue_mapping_mode dcb_q_mapping = DCB_VT_Q_MAPPING;
 173
 174 /*
 175  * Configurable number of RX/TX queues.
 176  */
 177 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 178 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 179
 180 /*
 181  * Configurable number of RX/TX ring descriptors.
 182  */
 183 #define RTE_TEST_RX_DESC_DEFAULT 128
 184 #define RTE_TEST_TX_DESC_DEFAULT 512
 185 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 186 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 187
 188 /*
 189  * Configurable values of RX and TX ring threshold registers.
 190  */
 191 #define RX_PTHRESH 8 /**< Default value of RX prefetch threshold register. */
 192 #define RX_HTHRESH 8 /**< Default value of RX host threshold register. */
 193 #define RX_WTHRESH 4 /**< Default value of RX write-back threshold register. */
 194
 195 #define TX_PTHRESH 36 /**< Default value of TX prefetch threshold register. */
 196 #define TX_HTHRESH 0 /**< Default value of TX host threshold register. */
 197 #define TX_WTHRESH 0 /**< Default value of TX write-back threshold register. */
 198
 199 struct rte_eth_thresh rx_thresh = {
 200         .pthresh = RX_PTHRESH,
 201         .hthresh = RX_HTHRESH,
 202         .wthresh = RX_WTHRESH,
 203 };
 204
 205 struct rte_eth_thresh tx_thresh = {
 206         .pthresh = TX_PTHRESH,
 207         .hthresh = TX_HTHRESH,
 208         .wthresh = TX_WTHRESH,
 209 };
 210
 211 /*
 212  * Configurable value of RX free threshold.
 213  */
 214 uint16_t rx_free_thresh = 0; /* Immediately free RX descriptors by default. */
 215
 216 /*
 217  * Configurable value of RX drop enable.
 218  */
 219 uint8_t rx_drop_en = 0; /* Drop packets when no descriptors for queue. */
 220
 221 /*
 222  * Configurable value of TX free threshold.
 223  */
 224 uint16_t tx_free_thresh = 0; /* Use default values. */
 225
 226 /*
 227  * Configurable value of TX RS bit threshold.
 228  */
 229 uint16_t tx_rs_thresh = 0; /* Use default values. */
 230
 231 /*
 232  * Configurable value of TX queue flags.
 233  */
 234 uint32_t txq_flags = 0; /* No flags set. */
 235
 236 /*
 237  * Receive Side Scaling (RSS) configuration.
 238  */
 239 uint16_t rss_hf = ETH_RSS_IPV4 | ETH_RSS_IPV6; /* RSS IP by default. */
 240
 241 /*
 242  * Port topology configuration
 243  */
 244 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 245
 246 /*
 247  * Ethernet device configuration.
 248  */
 249 struct rte_eth_rxmode rx_mode = {
 250         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 251         .split_hdr_size = 0,
 252         .header_split   = 0, /**< Header Split disabled. */
 253         .hw_ip_checksum = 0, /**< IP checksum offload disabled. */
 254         .hw_vlan_filter = 1, /**< VLAN filtering enabled. */
 255         .hw_vlan_strip  = 1, /**< VLAN strip enabled. */
 256         .hw_vlan_extend = 0, /**< Extended VLAN disabled. */
 257         .jumbo_frame    = 0, /**< Jumbo Frame Support disabled. */
 258         .hw_strip_crc   = 0, /**< CRC stripping by hardware disabled. */
 259 };
 260
 261 struct rte_fdir_conf fdir_conf = {
 262         .mode = RTE_FDIR_MODE_NONE,
 263         .pballoc = RTE_FDIR_PBALLOC_64K,
 264         .status = RTE_FDIR_REPORT_STATUS,
 265         .flexbytes_offset = 0x6,
 266         .drop_queue = 127,
 267 };
 268
 269 static volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 270
 271 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 272 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 273
 274 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 275 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 276
 277 uint16_t nb_tx_queue_stats_mappings = 0;
 278 uint16_t nb_rx_queue_stats_mappings = 0;
 279
 280 /* Forward function declarations */
 281 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port);
 282 static void check_all_ports_link_status(uint8_t port_num, uint32_t port_mask);
 283
 284 /*
 285  * Check if all the ports are started.
 286  * If yes, return positive value. If not, return zero.
 287  */
 288 static int all_ports_started(void);
 289
 290 /*
 291  * Setup default configuration.
 292  */
 293 static void
 294 set_default_fwd_lcores_config(void)
 295 {
 296         unsigned int i;
 297         unsigned int nb_lc;
 298
 299         nb_lc = 0;
 300         for (i = 0; i < RTE_MAX_LCORE; i++) {
 301                 if (! rte_lcore_is_enabled(i))
 302                         continue;
 303                 if (i == rte_get_master_lcore())
 304                         continue;
 305                 fwd_lcores_cpuids[nb_lc++] = i;
 306         }
 307         nb_lcores = (lcoreid_t) nb_lc;
 308         nb_cfg_lcores = nb_lcores;
 309         nb_fwd_lcores = 1;
 310 }
 311
 312 static void
 313 set_def_peer_eth_addrs(void)
 314 {
 315         portid_t i;
 316
 317         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 318                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 319                 peer_eth_addrs[i].addr_bytes[5] = i;
 320         }
 321 }
 322
 323 static void
 324 set_default_fwd_ports_config(void)
 325 {
 326         portid_t pt_id;
 327
 328         for (pt_id = 0; pt_id < nb_ports; pt_id++)
 329                 fwd_ports_ids[pt_id] = pt_id;
 330
 331         nb_cfg_ports = nb_ports;
 332         nb_fwd_ports = nb_ports;
 333 }
 334
 335 void
 336 set_def_fwd_config(void)
 337 {
 338         set_default_fwd_lcores_config();
 339         set_def_peer_eth_addrs();
 340         set_default_fwd_ports_config();
 341 }
 342
 343 /*
 344  * Configuration initialisation done once at init time.
 345  */
 346 struct mbuf_ctor_arg {
 347         uint16_t seg_buf_offset; /**< offset of data in data segment of mbuf. */
 348         uint16_t seg_buf_size;   /**< size of data segment in mbuf. */
 349 };
 350
 351 struct mbuf_pool_ctor_arg {
 352         uint16_t seg_buf_size; /**< size of data segment in mbuf. */
 353 };
 354
 355 static void
 356 testpmd_mbuf_ctor(struct rte_mempool *mp,
 357                   void *opaque_arg,
 358                   void *raw_mbuf,
 359                   __attribute__((unused)) unsigned i)
 360 {
 361         struct mbuf_ctor_arg *mb_ctor_arg;
 362         struct rte_mbuf    *mb;
 363
 364         mb_ctor_arg = (struct mbuf_ctor_arg *) opaque_arg;
 365         mb = (struct rte_mbuf *) raw_mbuf;
 366
 367         mb->type         = RTE_MBUF_PKT;
 368         mb->pool         = mp;
 369         mb->buf_addr     = (void *) ((char *)mb + mb_ctor_arg->seg_buf_offset);
 370         mb->buf_physaddr = (uint64_t) (rte_mempool_virt2phy(mp, mb) +
 371                         mb_ctor_arg->seg_buf_offset);
 372         mb->buf_len      = mb_ctor_arg->seg_buf_size;
 373         mb->type         = RTE_MBUF_PKT;
 374         mb->ol_flags     = 0;
 375         mb->pkt.data     = (char *) mb->buf_addr + RTE_PKTMBUF_HEADROOM;
 376         mb->pkt.nb_segs  = 1;
 377         mb->pkt.vlan_macip.data = 0;
 378         mb->pkt.hash.rss = 0;
 379 }
 380
 381 static void
 382 testpmd_mbuf_pool_ctor(struct rte_mempool *mp,
 383                        void *opaque_arg)
 384 {
 385         struct mbuf_pool_ctor_arg      *mbp_ctor_arg;
 386         struct rte_pktmbuf_pool_private *mbp_priv;
 387
 388         if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
 389                 printf("%s(%s) private_data_size %d < %d\n",
 390                        __func__, mp->name, (int) mp->private_data_size,
 391                        (int) sizeof(struct rte_pktmbuf_pool_private));
 392                 return;
 393         }
 394         mbp_ctor_arg = (struct mbuf_pool_ctor_arg *) opaque_arg;
 395         mbp_priv = (struct rte_pktmbuf_pool_private *)
 396                 ((char *)mp + sizeof(struct rte_mempool));
 397         mbp_priv->mbuf_data_room_size = mbp_ctor_arg->seg_buf_size;
 398 }
 399
 400 static void
 401 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 402                  unsigned int socket_id)
 403 {
 404         char pool_name[RTE_MEMPOOL_NAMESIZE];
 405         struct rte_mempool *rte_mp;
 406         struct mbuf_pool_ctor_arg mbp_ctor_arg;
 407         struct mbuf_ctor_arg mb_ctor_arg;
 408         uint32_t mb_size;
 409
 410         mbp_ctor_arg.seg_buf_size = (uint16_t) (RTE_PKTMBUF_HEADROOM +
 411                                                 mbuf_seg_size);
 412         mb_ctor_arg.seg_buf_offset =
 413                 (uint16_t) CACHE_LINE_ROUNDUP(sizeof(struct rte_mbuf));
 414         mb_ctor_arg.seg_buf_size = mbp_ctor_arg.seg_buf_size;
 415         mb_size = mb_ctor_arg.seg_buf_offset + mb_ctor_arg.seg_buf_size;
 416         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 417         rte_mp = rte_mempool_create(pool_name, nb_mbuf, (unsigned) mb_size,
 418                                     (unsigned) mb_mempool_cache,
 419                                     sizeof(struct rte_pktmbuf_pool_private),
 420                                     testpmd_mbuf_pool_ctor, &mbp_ctor_arg,
 421                                     testpmd_mbuf_ctor, &mb_ctor_arg,
 422                                     socket_id, 0);
 423         if (rte_mp == NULL) {
 424                 rte_exit(EXIT_FAILURE, "Creation of mbuf pool for socket %u "
 425                                                 "failed\n", socket_id);
 426         }
 427 }
 428
 429 static void
 430 init_config(void)
 431 {
 432         portid_t pid;
 433         struct rte_port *port;
 434         struct rte_mempool *mbp;
 435         unsigned int nb_mbuf_per_pool;
 436         lcoreid_t  lc_id;
 437         uint8_t port_per_socket[MAX_SOCKET];
 438
 439         memset(port_per_socket,0,MAX_SOCKET);
 440         /* Configuration of logical cores. */
 441         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
 442                                 sizeof(struct fwd_lcore *) * nb_lcores,
 443                                 CACHE_LINE_SIZE);
 444         if (fwd_lcores == NULL) {
 445                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
 446                                                         "failed\n", nb_lcores);
 447         }
 448         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 449                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
 450                                                sizeof(struct fwd_lcore),
 451                                                CACHE_LINE_SIZE);
 452                 if (fwd_lcores[lc_id] == NULL) {
 453                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
 454                                                                 "failed\n");
 455                 }
 456                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
 457         }
 458
 459         /*
 460          * Create pools of mbuf.
 461          * If NUMA support is disabled, create a single pool of mbuf in
 462          * socket 0 memory by default.
 463          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
 464          *
 465          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
 466          * nb_txd can be configured at run time.
 467          */
 468         if (param_total_num_mbufs)
 469                 nb_mbuf_per_pool = param_total_num_mbufs;
 470         else {
 471                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + (nb_lcores * mb_mempool_cache)
 472                                 + RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
 473
 474                 if (!numa_support)
 475                         nb_mbuf_per_pool = (nb_mbuf_per_pool * nb_ports);
 476         }
 477
 478         if (!numa_support) {
 479                 if (socket_num == UMA_NO_CONFIG)
 480                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0);
 481                 else
 482                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
 483                                                  socket_num);
 484         }
 485         /*
 486          * Records which Mbuf pool to use by each logical core, if needed.
 487          */
 488         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 489                 mbp = mbuf_pool_find(rte_lcore_to_socket_id(lc_id));
 490                 if (mbp == NULL)
 491                         mbp = mbuf_pool_find(0);
 492                 fwd_lcores[lc_id]->mbp = mbp;
 493         }
 494
 495         /* Configuration of Ethernet ports. */
 496         ports = rte_zmalloc("testpmd: ports",
 497                             sizeof(struct rte_port) * nb_ports,
 498                             CACHE_LINE_SIZE);
 499         if (ports == NULL) {
 500                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d struct rte_port) "
 501                                                         "failed\n", nb_ports);
 502         }
 503
 504         for (pid = 0; pid < nb_ports; pid++) {
 505                 port = &ports[pid];
 506                 rte_eth_dev_info_get(pid, &port->dev_info);
 507
 508                 if (numa_support) {
 509                         if (port_numa[pid] != NUMA_NO_CONFIG)
 510                                 port_per_socket[port_numa[pid]]++;
 511                         else {
 512                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
 513                                 port_per_socket[socket_id]++;
 514                         }
 515                 }
 516
 517                 /* set flag to initialize port/queue */
 518                 port->need_reconfig = 1;
 519                 port->need_reconfig_queues = 1;
 520         }
 521
 522         if (numa_support) {
 523                 uint8_t i;
 524                 unsigned int nb_mbuf;
 525
 526                 if (param_total_num_mbufs)
 527                         nb_mbuf_per_pool = nb_mbuf_per_pool/nb_ports;
 528
 529                 for (i = 0; i < MAX_SOCKET; i++) {
 530                         nb_mbuf = (nb_mbuf_per_pool *
 531                                                 port_per_socket[i]);
 532                         if (nb_mbuf)
 533                                 mbuf_pool_create(mbuf_data_size,
 534                                                 nb_mbuf,i);
 535                 }
 536         }
 537         init_port_config();
 538         /* Configuration of packet forwarding streams. */
 539         if (init_fwd_streams() < 0)
 540                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 541 }
 542
 543 int
 544 init_fwd_streams(void)
 545 {
 546         portid_t pid;
 547         struct rte_port *port;
 548         streamid_t sm_id, nb_fwd_streams_new;
 549
 550         /* set socket id according to numa or not */
 551         for (pid = 0; pid < nb_ports; pid++) {
 552                 port = &ports[pid];
 553                 if (nb_rxq > port->dev_info.max_rx_queues) {
 554                         printf("Fail: nb_rxq(%d) is greater than "
 555                                 "max_rx_queues(%d)\n", nb_rxq,
 556                                 port->dev_info.max_rx_queues);
 557                         return -1;
 558                 }
 559                 if (nb_txq > port->dev_info.max_tx_queues) {
 560                         printf("Fail: nb_txq(%d) is greater than "
 561                                 "max_tx_queues(%d)\n", nb_txq,
 562                                 port->dev_info.max_tx_queues);
 563                         return -1;
 564                 }
 565                 if (numa_support)
 566                         port->socket_id = rte_eth_dev_socket_id(pid);
 567                 else {
 568                         if (socket_num == UMA_NO_CONFIG)
 569                                 port->socket_id = 0;
 570                         else
 571                                 port->socket_id = socket_num;
 572                 }
 573         }
 574
 575         nb_fwd_streams_new = (streamid_t)(nb_ports * nb_rxq);
 576         if (nb_fwd_streams_new == nb_fwd_streams)
 577                 return 0;
 578         /* clear the old */
 579         if (fwd_streams != NULL) {
 580                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 581                         if (fwd_streams[sm_id] == NULL)
 582                                 continue;
 583                         rte_free(fwd_streams[sm_id]);
 584                         fwd_streams[sm_id] = NULL;
 585                 }
 586                 rte_free(fwd_streams);
 587                 fwd_streams = NULL;
 588         }
 589
 590         /* init new */
 591         nb_fwd_streams = nb_fwd_streams_new;
 592         fwd_streams = rte_zmalloc("testpmd: fwd_streams",
 593                 sizeof(struct fwd_stream *) * nb_fwd_streams, CACHE_LINE_SIZE);
 594         if (fwd_streams == NULL)
 595                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_stream *)) "
 596                                                 "failed\n", nb_fwd_streams);
 597
 598         for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 599                 fwd_streams[sm_id] = rte_zmalloc("testpmd: struct fwd_stream",
 600                                 sizeof(struct fwd_stream), CACHE_LINE_SIZE);
 601                 if (fwd_streams[sm_id] == NULL)
 602                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_stream)"
 603                                                                 " failed\n");
 604         }
 605
 606         return 0;
 607 }
 608
 609 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 610 static void
 611 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
 612 {
 613         unsigned int total_burst;
 614         unsigned int nb_burst;
 615         unsigned int burst_stats[3];
 616         uint16_t pktnb_stats[3];
 617         uint16_t nb_pkt;
 618         int burst_percent[3];
 619
 620         /*
 621          * First compute the total number of packet bursts and the
 622          * two highest numbers of bursts of the same number of packets.
 623          */
 624         total_burst = 0;
 625         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
 626         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
 627         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
 628                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
 629                 if (nb_burst == 0)
 630                         continue;
 631                 total_burst += nb_burst;
 632                 if (nb_burst > burst_stats[0]) {
 633                         burst_stats[1] = burst_stats[0];
 634                         pktnb_stats[1] = pktnb_stats[0];
 635                         burst_stats[0] = nb_burst;
 636                         pktnb_stats[0] = nb_pkt;
 637                 }
 638         }
 639         if (total_burst == 0)
 640                 return;
 641         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
 642         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
 643                burst_percent[0], (int) pktnb_stats[0]);
 644         if (burst_stats[0] == total_burst) {
 645                 printf("]\n");
 646                 return;
 647         }
 648         if (burst_stats[0] + burst_stats[1] == total_burst) {
 649                 printf(" + %d%% of %d pkts]\n",
 650                        100 - burst_percent[0], pktnb_stats[1]);
 651                 return;
 652         }
 653         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
 654         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
 655         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
 656                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
 657                 return;
 658         }
 659         printf(" + %d%% of %d pkts + %d%% of others]\n",
 660                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
 661 }
 662 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
 663
 664 static void
 665 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats)
 666 {
 667         struct rte_port *port;
 668         uint8_t i;
 669
 670         static const char *fwd_stats_border = "----------------------";
 671
 672         port = &ports[port_id];
 673         printf("\n  %s Forward statistics for port %-2d %s\n",
 674                fwd_stats_border, port_id, fwd_stats_border);
 675
 676         if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) {
 677                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
 678                        "%-"PRIu64"\n",
 679                        stats->ipackets, stats->ierrors,
 680                        (uint64_t) (stats->ipackets + stats->ierrors));
 681
 682                 if (cur_fwd_eng == &csum_fwd_engine)
 683                         printf("  Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n",
 684                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 685
 686                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
 687                        "%-"PRIu64"\n",
 688                        stats->opackets, port->tx_dropped,
 689                        (uint64_t) (stats->opackets + port->tx_dropped));
 690
 691                 if (stats->rx_nombuf > 0)
 692                         printf("  RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf);
 693
 694         }
 695         else {
 696                 printf("  RX-packets:             %14"PRIu64"    RX-dropped:%14"PRIu64"    RX-total:"
 697                        "%14"PRIu64"\n",
 698                        stats->ipackets, stats->ierrors,
 699                        (uint64_t) (stats->ipackets + stats->ierrors));
 700
 701                 if (cur_fwd_eng == &csum_fwd_engine)
 702                         printf("  Bad-ipcsum:%14"PRIu64"    Bad-l4csum:%14"PRIu64"\n",
 703                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 704
 705                 printf("  TX-packets:             %14"PRIu64"    TX-dropped:%14"PRIu64"    TX-total:"
 706                        "%14"PRIu64"\n",
 707                        stats->opackets, port->tx_dropped,
 708                        (uint64_t) (stats->opackets + port->tx_dropped));
 709
 710                 if (stats->rx_nombuf > 0)
 711                         printf("  RX-nombufs:%14"PRIu64"\n", stats->rx_nombuf);
 712         }
 713 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 714         if (port->rx_stream)
 715                 pkt_burst_stats_display("RX",
 716                         &port->rx_stream->rx_burst_stats);
 717         if (port->tx_stream)
 718                 pkt_burst_stats_display("TX",
 719                         &port->tx_stream->tx_burst_stats);
 720 #endif
 721         /* stats fdir */
 722         if (fdir_conf.mode != RTE_FDIR_MODE_NONE)
 723                 printf("  Fdirmiss:%14"PRIu64"    Fdirmatch:%14"PRIu64"\n",
 724                        stats->fdirmiss,
 725                        stats->fdirmatch);
 726
 727         if (port->rx_queue_stats_mapping_enabled) {
 728                 printf("\n");
 729                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 730                         printf("  Stats reg %2d RX-packets:%14"PRIu64
 731                                "     RX-errors:%14"PRIu64
 732                                "    RX-bytes:%14"PRIu64"\n",
 733                                i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]);
 734                 }
 735                 printf("\n");
 736         }
 737         if (port->tx_queue_stats_mapping_enabled) {
 738                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 739                         printf("  Stats reg %2d TX-packets:%14"PRIu64
 740                                "                                 TX-bytes:%14"PRIu64"\n",
 741                                i, stats->q_opackets[i], stats->q_obytes[i]);
 742                 }
 743         }
 744
 745         printf("  %s--------------------------------%s\n",
 746                fwd_stats_border, fwd_stats_border);
 747 }
 748
 749 static void
 750 fwd_stream_stats_display(streamid_t stream_id)
 751 {
 752         struct fwd_stream *fs;
 753         static const char *fwd_top_stats_border = "-------";
 754
 755         fs = fwd_streams[stream_id];
 756         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
 757             (fs->fwd_dropped == 0))
 758                 return;
 759         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
 760                "TX Port=%2d/Queue=%2d %s\n",
 761                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
 762                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
 763         printf("  RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u",
 764                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
 765
 766         /* if checksum mode */
 767         if (cur_fwd_eng == &csum_fwd_engine) {
 768                printf("  RX- bad IP checksum: %-14u  Rx- bad L4 checksum: "
 769                         "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum);
 770         }
 771
 772 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 773         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
 774         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
 775 #endif
 776 }
 777
 778 static void
 779 flush_all_rx_queues(void)
 780 {
 781         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 782         portid_t  rxp;
 783         queueid_t rxq;
 784         uint16_t  nb_rx;
 785         uint16_t  i;
 786         uint8_t   j;
 787
 788         for (j = 0; j < 2; j++) {
 789                 for (rxp = 0; rxp < nb_ports; rxp++) {
 790                         for (rxq = 0; rxq < nb_rxq; rxq++) {
 791                                 do {
 792                                         nb_rx = rte_eth_rx_burst(rxp, rxq,
 793                                                 pkts_burst, MAX_PKT_BURST);
 794                                         for (i = 0; i < nb_rx; i++)
 795                                                 rte_pktmbuf_free(pkts_burst[i]);
 796                                 } while (nb_rx > 0);
 797                         }
 798                 }
 799                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
 800         }
 801 }
 802
 803 static void
 804 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
 805 {
 806         struct fwd_stream **fsm;
 807         streamid_t nb_fs;
 808         streamid_t sm_id;
 809
 810         fsm = &fwd_streams[fc->stream_idx];
 811         nb_fs = fc->stream_nb;
 812         do {
 813                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
 814                         (*pkt_fwd)(fsm[sm_id]);
 815         } while (! fc->stopped);
 816 }
 817
 818 static int
 819 start_pkt_forward_on_core(void *fwd_arg)
 820 {
 821         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
 822                              cur_fwd_config.fwd_eng->packet_fwd);
 823         return 0;
 824 }
 825
 826 /*
 827  * Run the TXONLY packet forwarding engine to send a single burst of packets.
 828  * Used to start communication flows in network loopback test configurations.
 829  */
 830 static int
 831 run_one_txonly_burst_on_core(void *fwd_arg)
 832 {
 833         struct fwd_lcore *fwd_lc;
 834         struct fwd_lcore tmp_lcore;
 835
 836         fwd_lc = (struct fwd_lcore *) fwd_arg;
 837         tmp_lcore = *fwd_lc;
 838         tmp_lcore.stopped = 1;
 839         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
 840         return 0;
 841 }
 842
 843 /*
 844  * Launch packet forwarding:
 845  *     - Setup per-port forwarding context.
 846  *     - launch logical cores with their forwarding configuration.
 847  */
 848 static void
 849 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
 850 {
 851         port_fwd_begin_t port_fwd_begin;
 852         unsigned int i;
 853         unsigned int lc_id;
 854         int diag;
 855
 856         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
 857         if (port_fwd_begin != NULL) {
 858                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
 859                         (*port_fwd_begin)(fwd_ports_ids[i]);
 860         }
 861         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
 862                 lc_id = fwd_lcores_cpuids[i];
 863                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
 864                         fwd_lcores[i]->stopped = 0;
 865                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
 866                                                      fwd_lcores[i], lc_id);
 867                         if (diag != 0)
 868                                 printf("launch lcore %u failed - diag=%d\n",
 869                                        lc_id, diag);
 870                 }
 871         }
 872 }
 873
 874 /*
 875  * Launch packet forwarding configuration.
 876  */
 877 void
 878 start_packet_forwarding(int with_tx_first)
 879 {
 880         port_fwd_begin_t port_fwd_begin;
 881         port_fwd_end_t  port_fwd_end;
 882         struct rte_port *port;
 883         unsigned int i;
 884         portid_t   pt_id;
 885         streamid_t sm_id;
 886
 887         if (all_ports_started() == 0) {
 888                 printf("Not all ports were started\n");
 889                 return;
 890         }
 891         if (test_done == 0) {
 892                 printf("Packet forwarding already started\n");
 893                 return;
 894         }
 895         if((dcb_test) && (nb_fwd_lcores == 1)) {
 896                 printf("In DCB mode,the nb forwarding cores should be larger than 1.\n");
 897                 return;
 898         }
 899         test_done = 0;
 900         flush_all_rx_queues();
 901         fwd_config_setup();
 902         rxtx_config_display();
 903
 904         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
 905                 pt_id = fwd_ports_ids[i];
 906                 port = &ports[pt_id];
 907                 rte_eth_stats_get(pt_id, &port->stats);
 908                 port->tx_dropped = 0;
 909
 910                 map_port_queue_stats_mapping_registers(pt_id, port);
 911         }
 912         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 913                 fwd_streams[sm_id]->rx_packets = 0;
 914                 fwd_streams[sm_id]->tx_packets = 0;
 915                 fwd_streams[sm_id]->fwd_dropped = 0;
 916                 fwd_streams[sm_id]->rx_bad_ip_csum = 0;
 917                 fwd_streams[sm_id]->rx_bad_l4_csum = 0;
 918
 919 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 920                 memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
 921                        sizeof(fwd_streams[sm_id]->rx_burst_stats));
 922                 memset(&fwd_streams[sm_id]->tx_burst_stats, 0,
 923                        sizeof(fwd_streams[sm_id]->tx_burst_stats));
 924 #endif
 925 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 926                 fwd_streams[sm_id]->core_cycles = 0;
 927 #endif
 928         }
 929         if (with_tx_first) {
 930                 port_fwd_begin = tx_only_engine.port_fwd_begin;
 931                 if (port_fwd_begin != NULL) {
 932                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
 933                                 (*port_fwd_begin)(fwd_ports_ids[i]);
 934                 }
 935                 launch_packet_forwarding(run_one_txonly_burst_on_core);
 936                 rte_eal_mp_wait_lcore();
 937                 port_fwd_end = tx_only_engine.port_fwd_end;
 938                 if (port_fwd_end != NULL) {
 939                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
 940                                 (*port_fwd_end)(fwd_ports_ids[i]);
 941                 }
 942         }
 943         launch_packet_forwarding(start_pkt_forward_on_core);
 944 }
 945
 946 void
 947 stop_packet_forwarding(void)
 948 {
 949         struct rte_eth_stats stats;
 950         struct rte_port *port;
 951         port_fwd_end_t  port_fwd_end;
 952         int i;
 953         portid_t   pt_id;
 954         streamid_t sm_id;
 955         lcoreid_t  lc_id;
 956         uint64_t total_recv;
 957         uint64_t total_xmit;
 958         uint64_t total_rx_dropped;
 959         uint64_t total_tx_dropped;
 960         uint64_t total_rx_nombuf;
 961         uint64_t tx_dropped;
 962         uint64_t rx_bad_ip_csum;
 963         uint64_t rx_bad_l4_csum;
 964 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 965         uint64_t fwd_cycles;
 966 #endif
 967         static const char *acc_stats_border = "+++++++++++++++";
 968
 969         if (all_ports_started() == 0) {
 970                 printf("Not all ports were started\n");
 971                 return;
 972         }
 973         if (test_done) {
 974                 printf("Packet forwarding not started\n");
 975                 return;
 976         }
 977         printf("Telling cores to stop...");
 978         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
 979                 fwd_lcores[lc_id]->stopped = 1;
 980         printf("\nWaiting for lcores to finish...\n");
 981         rte_eal_mp_wait_lcore();
 982         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
 983         if (port_fwd_end != NULL) {
 984                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
 985                         pt_id = fwd_ports_ids[i];
 986                         (*port_fwd_end)(pt_id);
 987                 }
 988         }
 989 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 990         fwd_cycles = 0;
 991 #endif
 992         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 993                 if (cur_fwd_config.nb_fwd_streams >
 994                     cur_fwd_config.nb_fwd_ports) {
 995                         fwd_stream_stats_display(sm_id);
 996                         ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL;
 997                         ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL;
 998                 } else {
 999                         ports[fwd_streams[sm_id]->tx_port].tx_stream =
1000                                 fwd_streams[sm_id];
1001                         ports[fwd_streams[sm_id]->rx_port].rx_stream =
1002                                 fwd_streams[sm_id];
1003                 }
1004                 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped;
1005                 tx_dropped = (uint64_t) (tx_dropped +
1006                                          fwd_streams[sm_id]->fwd_dropped);
1007                 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped;
1008
1009                 rx_bad_ip_csum =
1010                         ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum;
1011                 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum +
1012                                          fwd_streams[sm_id]->rx_bad_ip_csum);
1013                 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum =
1014                                                         rx_bad_ip_csum;
1015
1016                 rx_bad_l4_csum =
1017                         ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum;
1018                 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum +
1019                                          fwd_streams[sm_id]->rx_bad_l4_csum);
1020                 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum =
1021                                                         rx_bad_l4_csum;
1022
1023 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1024                 fwd_cycles = (uint64_t) (fwd_cycles +
1025                                          fwd_streams[sm_id]->core_cycles);
1026 #endif
1027         }
1028         total_recv = 0;
1029         total_xmit = 0;
1030         total_rx_dropped = 0;
1031         total_tx_dropped = 0;
1032         total_rx_nombuf  = 0;
1033         for (i = 0; i < ((cur_fwd_config.nb_fwd_ports + 1) & ~0x1); i++) {
1034                 pt_id = fwd_ports_ids[i];
1035
1036                 port = &ports[pt_id];
1037                 rte_eth_stats_get(pt_id, &stats);
1038                 stats.ipackets -= port->stats.ipackets;
1039                 port->stats.ipackets = 0;
1040                 stats.opackets -= port->stats.opackets;
1041                 port->stats.opackets = 0;
1042                 stats.ibytes   -= port->stats.ibytes;
1043                 port->stats.ibytes = 0;
1044                 stats.obytes   -= port->stats.obytes;
1045                 port->stats.obytes = 0;
1046                 stats.ierrors  -= port->stats.ierrors;
1047                 port->stats.ierrors = 0;
1048                 stats.oerrors  -= port->stats.oerrors;
1049                 port->stats.oerrors = 0;
1050                 stats.rx_nombuf -= port->stats.rx_nombuf;
1051                 port->stats.rx_nombuf = 0;
1052                 stats.fdirmatch -= port->stats.fdirmatch;
1053                 port->stats.rx_nombuf = 0;
1054                 stats.fdirmiss -= port->stats.fdirmiss;
1055                 port->stats.rx_nombuf = 0;
1056
1057                 total_recv += stats.ipackets;
1058                 total_xmit += stats.opackets;
1059                 total_rx_dropped += stats.ierrors;
1060                 total_tx_dropped += port->tx_dropped;
1061                 total_rx_nombuf  += stats.rx_nombuf;
1062
1063                 fwd_port_stats_display(pt_id, &stats);
1064         }
1065         printf("\n  %s Accumulated forward statistics for all ports"
1066                "%s\n",
1067                acc_stats_border, acc_stats_border);
1068         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1069                "%-"PRIu64"\n"
1070                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1071                "%-"PRIu64"\n",
1072                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1073                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1074         if (total_rx_nombuf > 0)
1075                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1076         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1077                "%s\n",
1078                acc_stats_border, acc_stats_border);
1079 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1080         if (total_recv > 0)
1081                 printf("\n  CPU cycles/packet=%u (total cycles="
1082                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1083                        (unsigned int)(fwd_cycles / total_recv),
1084                        fwd_cycles, total_recv);
1085 #endif
1086         printf("\nDone.\n");
1087         test_done = 1;
1088 }
1089
1090 static int
1091 all_ports_started(void)
1092 {
1093         portid_t pi;
1094         struct rte_port *port;
1095
1096         for (pi = 0; pi < nb_ports; pi++) {
1097                 port = &ports[pi];
1098                 /* Check if there is a port which is not started */
1099                 if (port->port_status != RTE_PORT_STARTED)
1100                         return 0;
1101         }
1102
1103         /* No port is not started */
1104         return 1;
1105 }
1106
1107 void
1108 start_port(portid_t pid)
1109 {
1110         int diag, need_check_link_status = 0;
1111         portid_t pi;
1112         queueid_t qi;
1113         struct rte_port *port;
1114
1115         if (test_done == 0) {
1116                 printf("Please stop forwarding first\n");
1117                 return;
1118         }
1119
1120         if (init_fwd_streams() < 0) {
1121                 printf("Fail from init_fwd_streams()\n");
1122                 return;
1123         }
1124
1125         if(dcb_config)
1126                 dcb_test = 1;
1127         for (pi = 0; pi < nb_ports; pi++) {
1128                 if (pid < nb_ports && pid != pi)
1129                         continue;
1130
1131                 port = &ports[pi];
1132                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1133                                                  RTE_PORT_HANDLING) == 0) {
1134                         printf("Port %d is now not stopped\n", pi);
1135                         continue;
1136                 }
1137
1138                 if (port->need_reconfig > 0) {
1139                         port->need_reconfig = 0;
1140
1141                         printf("Configuring Port %d (socket %d)\n", pi,
1142                                         rte_eth_dev_socket_id(pi));
1143                         /* configure port */
1144                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1145                                                 &(port->dev_conf));
1146                         if (diag != 0) {
1147                                 if (rte_atomic16_cmpset(&(port->port_status),
1148                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1149                                         printf("Port %d can not be set back "
1150                                                         "to stopped\n", pi);
1151                                 printf("Fail to configure port %d\n", pi);
1152                                 /* try to reconfigure port next time */
1153                                 port->need_reconfig = 1;
1154                                 return;
1155                         }
1156                 }
1157                 if (port->need_reconfig_queues > 0) {
1158                         port->need_reconfig_queues = 0;
1159                         /* setup tx queues */
1160                         for (qi = 0; qi < nb_txq; qi++) {
1161                                 if ((numa_support) &&
1162                                         (txring_numa[pi] != NUMA_NO_CONFIG))
1163                                         diag = rte_eth_tx_queue_setup(pi, qi,
1164                                                 nb_txd,txring_numa[pi],
1165                                                 &(port->tx_conf));
1166                                 else
1167                                         diag = rte_eth_tx_queue_setup(pi, qi,
1168                                                 nb_txd,port->socket_id,
1169                                                 &(port->tx_conf));
1170
1171                                 if (diag == 0)
1172                                         continue;
1173
1174                                 /* Fail to setup tx queue, return */
1175                                 if (rte_atomic16_cmpset(&(port->port_status),
1176                                                         RTE_PORT_HANDLING,
1177                                                         RTE_PORT_STOPPED) == 0)
1178                                         printf("Port %d can not be set back "
1179                                                         "to stopped\n", pi);
1180                                 printf("Fail to configure port %d tx queues\n", pi);
1181                                 /* try to reconfigure queues next time */
1182                                 port->need_reconfig_queues = 1;
1183                                 return;
1184                         }
1185                         /* setup rx queues */
1186                         for (qi = 0; qi < nb_rxq; qi++) {
1187                                 if ((numa_support) &&
1188                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
1189                                         struct rte_mempool * mp =
1190                                                 mbuf_pool_find(rxring_numa[pi]);
1191                                         if (mp == NULL) {
1192                                                 printf("Failed to setup RX queue:"
1193                                                         "No mempool allocation"
1194                                                         "on the socket %d\n",
1195                                                         rxring_numa[pi]);
1196                                                 return;
1197                                         }
1198
1199                                         diag = rte_eth_rx_queue_setup(pi, qi,
1200                                              nb_rxd,rxring_numa[pi],
1201                                              &(port->rx_conf),mp);
1202                                 }
1203                                 else
1204                                         diag = rte_eth_rx_queue_setup(pi, qi,
1205                                              nb_rxd,port->socket_id,
1206                                              &(port->rx_conf),
1207                                              mbuf_pool_find(port->socket_id));
1208
1209                                 if (diag == 0)
1210                                         continue;
1211
1212
1213                                 /* Fail to setup rx queue, return */
1214                                 if (rte_atomic16_cmpset(&(port->port_status),
1215                                                         RTE_PORT_HANDLING,
1216                                                         RTE_PORT_STOPPED) == 0)
1217                                         printf("Port %d can not be set back "
1218                                                         "to stopped\n", pi);
1219                                 printf("Fail to configure port %d rx queues\n", pi);
1220                                 /* try to reconfigure queues next time */
1221                                 port->need_reconfig_queues = 1;
1222                                 return;
1223                         }
1224                 }
1225                 /* start port */
1226                 if (rte_eth_dev_start(pi) < 0) {
1227                         printf("Fail to start port %d\n", pi);
1228
1229                         /* Fail to setup rx queue, return */
1230                         if (rte_atomic16_cmpset(&(port->port_status),
1231                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1232                                 printf("Port %d can not be set back to "
1233                                                         "stopped\n", pi);
1234                         continue;
1235                 }
1236
1237                 if (rte_atomic16_cmpset(&(port->port_status),
1238                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
1239                         printf("Port %d can not be set into started\n", pi);
1240
1241                 /* at least one port started, need checking link status */
1242                 need_check_link_status = 1;
1243         }
1244
1245         if (need_check_link_status)
1246                 check_all_ports_link_status(nb_ports, RTE_PORT_ALL);
1247         else
1248                 printf("Please stop the ports first\n");
1249
1250         printf("Done\n");
1251 }
1252
1253 void
1254 stop_port(portid_t pid)
1255 {
1256         portid_t pi;
1257         struct rte_port *port;
1258         int need_check_link_status = 0;
1259
1260         if (test_done == 0) {
1261                 printf("Please stop forwarding first\n");
1262                 return;
1263         }
1264         if (dcb_test) {
1265                 dcb_test = 0;
1266                 dcb_config = 0;
1267         }
1268         printf("Stopping ports...\n");
1269
1270         for (pi = 0; pi < nb_ports; pi++) {
1271                 if (pid < nb_ports && pid != pi)
1272                         continue;
1273
1274                 port = &ports[pi];
1275                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
1276                                                 RTE_PORT_HANDLING) == 0)
1277                         continue;
1278
1279                 rte_eth_dev_stop(pi);
1280
1281                 if (rte_atomic16_cmpset(&(port->port_status),
1282                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1283                         printf("Port %d can not be set into stopped\n", pi);
1284                 need_check_link_status = 1;
1285         }
1286         if (need_check_link_status)
1287                 check_all_ports_link_status(nb_ports, RTE_PORT_ALL);
1288
1289         printf("Done\n");
1290 }
1291
1292 void
1293 close_port(portid_t pid)
1294 {
1295         portid_t pi;
1296         struct rte_port *port;
1297
1298         if (test_done == 0) {
1299                 printf("Please stop forwarding first\n");
1300                 return;
1301         }
1302
1303         printf("Closing ports...\n");
1304
1305         for (pi = 0; pi < nb_ports; pi++) {
1306                 if (pid < nb_ports && pid != pi)
1307                         continue;
1308
1309                 port = &ports[pi];
1310                 if (rte_atomic16_cmpset(&(port->port_status),
1311                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
1312                         printf("Port %d is now not stopped\n", pi);
1313                         continue;
1314                 }
1315
1316                 rte_eth_dev_close(pi);
1317
1318                 if (rte_atomic16_cmpset(&(port->port_status),
1319                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
1320                         printf("Port %d can not be set into stopped\n", pi);
1321         }
1322
1323         printf("Done\n");
1324 }
1325
1326 int
1327 all_ports_stopped(void)
1328 {
1329         portid_t pi;
1330         struct rte_port *port;
1331
1332         for (pi = 0; pi < nb_ports; pi++) {
1333                 port = &ports[pi];
1334                 if (port->port_status != RTE_PORT_STOPPED)
1335                         return 0;
1336         }
1337
1338         return 1;
1339 }
1340
1341 void
1342 pmd_test_exit(void)
1343 {
1344         portid_t pt_id;
1345
1346         for (pt_id = 0; pt_id < nb_ports; pt_id++) {
1347                 printf("Stopping port %d...", pt_id);
1348                 fflush(stdout);
1349                 rte_eth_dev_close(pt_id);
1350                 printf("done\n");
1351         }
1352         printf("bye...\n");
1353 }
1354
1355 typedef void (*cmd_func_t)(void);
1356 struct pmd_test_command {
1357         const char *cmd_name;
1358         cmd_func_t cmd_func;
1359 };
1360
1361 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
1362
1363 /* Check the link status of all ports in up to 9s, and print them finally */
1364 static void
1365 check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
1366 {
1367 #define CHECK_INTERVAL 100 /* 100ms */
1368 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
1369         uint8_t portid, count, all_ports_up, print_flag = 0;
1370         struct rte_eth_link link;
1371
1372         printf("Checking link statuses...\n");
1373         fflush(stdout);
1374         for (count = 0; count <= MAX_CHECK_TIME; count++) {
1375                 all_ports_up = 1;
1376                 for (portid = 0; portid < port_num; portid++) {
1377                         if ((port_mask & (1 << portid)) == 0)
1378                                 continue;
1379                         memset(&link, 0, sizeof(link));
1380                         rte_eth_link_get_nowait(portid, &link);
1381                         /* print link status if flag set */
1382                         if (print_flag == 1) {
1383                                 if (link.link_status)
1384                                         printf("Port %d Link Up - speed %u "
1385                                                 "Mbps - %s\n", (uint8_t)portid,
1386                                                 (unsigned)link.link_speed,
1387                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
1388                                         ("full-duplex") : ("half-duplex\n"));
1389                                 else
1390                                         printf("Port %d Link Down\n",
1391                                                 (uint8_t)portid);
1392                                 continue;
1393                         }
1394                         /* clear all_ports_up flag if any link down */
1395                         if (link.link_status == 0) {
1396                                 all_ports_up = 0;
1397                                 break;
1398                         }
1399                 }
1400                 /* after finally printing all link status, get out */
1401                 if (print_flag == 1)
1402                         break;
1403
1404                 if (all_ports_up == 0) {
1405                         fflush(stdout);
1406                         rte_delay_ms(CHECK_INTERVAL);
1407                 }
1408
1409                 /* set the print_flag if all ports up or timeout */
1410                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
1411                         print_flag = 1;
1412                 }
1413         }
1414 }
1415
1416 static int
1417 set_tx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port)
1418 {
1419         uint16_t i;
1420         int diag;
1421         uint8_t mapping_found = 0;
1422
1423         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
1424                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
1425                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
1426                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
1427                                         tx_queue_stats_mappings[i].queue_id,
1428                                         tx_queue_stats_mappings[i].stats_counter_id);
1429                         if (diag != 0)
1430                                 return diag;
1431                         mapping_found = 1;
1432                 }
1433         }
1434         if (mapping_found)
1435                 port->tx_queue_stats_mapping_enabled = 1;
1436         return 0;
1437 }
1438
1439 static int
1440 set_rx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port)
1441 {
1442         uint16_t i;
1443         int diag;
1444         uint8_t mapping_found = 0;
1445
1446         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
1447                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
1448                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
1449                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
1450                                         rx_queue_stats_mappings[i].queue_id,
1451                                         rx_queue_stats_mappings[i].stats_counter_id);
1452                         if (diag != 0)
1453                                 return diag;
1454                         mapping_found = 1;
1455                 }
1456         }
1457         if (mapping_found)
1458                 port->rx_queue_stats_mapping_enabled = 1;
1459         return 0;
1460 }
1461
1462 static void
1463 map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port)
1464 {
1465         int diag = 0;
1466
1467         diag = set_tx_queue_stats_mapping_registers(pi, port);
1468         if (diag != 0) {
1469                 if (diag == -ENOTSUP) {
1470                         port->tx_queue_stats_mapping_enabled = 0;
1471                         printf("TX queue stats mapping not supported port id=%d\n", pi);
1472                 }
1473                 else
1474                         rte_exit(EXIT_FAILURE,
1475                                         "set_tx_queue_stats_mapping_registers "
1476                                         "failed for port id=%d diag=%d\n",
1477                                         pi, diag);
1478         }
1479
1480         diag = set_rx_queue_stats_mapping_registers(pi, port);
1481         if (diag != 0) {
1482                 if (diag == -ENOTSUP) {
1483                         port->rx_queue_stats_mapping_enabled = 0;
1484                         printf("RX queue stats mapping not supported port id=%d\n", pi);
1485                 }
1486                 else
1487                         rte_exit(EXIT_FAILURE,
1488                                         "set_rx_queue_stats_mapping_registers "
1489                                         "failed for port id=%d diag=%d\n",
1490                                         pi, diag);
1491         }
1492 }
1493
1494 void
1495 init_port_config(void)
1496 {
1497         portid_t pid;
1498         struct rte_port *port;
1499
1500         for (pid = 0; pid < nb_ports; pid++) {
1501                 port = &ports[pid];
1502                 port->dev_conf.rxmode = rx_mode;
1503                 port->dev_conf.fdir_conf = fdir_conf;
1504                 if (nb_rxq > 0) {
1505                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1506                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = rss_hf;
1507                 } else {
1508                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1509                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
1510                 }
1511                 port->rx_conf.rx_thresh = rx_thresh;
1512                 port->rx_conf.rx_free_thresh = rx_free_thresh;
1513                 port->rx_conf.rx_drop_en = rx_drop_en;
1514                 port->tx_conf.tx_thresh = tx_thresh;
1515                 port->tx_conf.tx_rs_thresh = tx_rs_thresh;
1516                 port->tx_conf.tx_free_thresh = tx_free_thresh;
1517                 port->tx_conf.txq_flags = txq_flags;
1518
1519                 rte_eth_macaddr_get(pid, &port->eth_addr);
1520
1521                 map_port_queue_stats_mapping_registers(pid, port);
1522         }
1523 }
1524
1525 const uint16_t vlan_tags[] = {
1526                 0,  1,  2,  3,  4,  5,  6,  7,
1527                 8,  9, 10, 11,  12, 13, 14, 15,
1528                 16, 17, 18, 19, 20, 21, 22, 23,
1529                 24, 25, 26, 27, 28, 29, 30, 31
1530 };
1531
1532 static  int
1533 get_eth_dcb_conf(struct rte_eth_conf *eth_conf, struct dcb_config *dcb_conf)
1534 {
1535         uint8_t i;
1536
1537         /*
1538          * Builds up the correct configuration for dcb+vt based on the vlan tags array
1539          * given above, and the number of traffic classes available for use.
1540          */
1541         if (dcb_conf->dcb_mode == DCB_VT_ENABLED) {
1542                 struct rte_eth_vmdq_dcb_conf vmdq_rx_conf;
1543                 struct rte_eth_vmdq_dcb_tx_conf vmdq_tx_conf;
1544
1545                 /* VMDQ+DCB RX and TX configrations */
1546                 vmdq_rx_conf.enable_default_pool = 0;
1547                 vmdq_rx_conf.default_pool = 0;
1548                 vmdq_rx_conf.nb_queue_pools =
1549                         (dcb_conf->num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
1550                 vmdq_tx_conf.nb_queue_pools =
1551                         (dcb_conf->num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
1552
1553                 vmdq_rx_conf.nb_pool_maps = sizeof( vlan_tags )/sizeof( vlan_tags[ 0 ]);
1554                 for (i = 0; i < vmdq_rx_conf.nb_pool_maps; i++) {
1555                         vmdq_rx_conf.pool_map[i].vlan_id = vlan_tags[ i ];
1556                         vmdq_rx_conf.pool_map[i].pools = 1 << (i % vmdq_rx_conf.nb_queue_pools);
1557                 }
1558                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
1559                         vmdq_rx_conf.dcb_queue[i] = i;
1560                         vmdq_tx_conf.dcb_queue[i] = i;
1561                 }
1562
1563                 /*set DCB mode of RX and TX of multiple queues*/
1564                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
1565                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
1566                 if (dcb_conf->pfc_en)
1567                         eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT|ETH_DCB_PFC_SUPPORT;
1568                 else
1569                         eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
1570
1571                 (void)(rte_memcpy(&eth_conf->rx_adv_conf.vmdq_dcb_conf, &vmdq_rx_conf,
1572                                 sizeof(struct rte_eth_vmdq_dcb_conf)));
1573                 (void)(rte_memcpy(&eth_conf->tx_adv_conf.vmdq_dcb_tx_conf, &vmdq_tx_conf,
1574                                 sizeof(struct rte_eth_vmdq_dcb_tx_conf)));
1575         }
1576         else {
1577                 struct rte_eth_dcb_rx_conf rx_conf;
1578                 struct rte_eth_dcb_tx_conf tx_conf;
1579
1580                 /* queue mapping configuration of DCB RX and TX */
1581                 if (dcb_conf->num_tcs == ETH_4_TCS)
1582                         dcb_q_mapping = DCB_4_TCS_Q_MAPPING;
1583                 else
1584                         dcb_q_mapping = DCB_8_TCS_Q_MAPPING;
1585
1586                 rx_conf.nb_tcs = dcb_conf->num_tcs;
1587                 tx_conf.nb_tcs = dcb_conf->num_tcs;
1588
1589                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++){
1590                         rx_conf.dcb_queue[i] = i;
1591                         tx_conf.dcb_queue[i] = i;
1592                 }
1593                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB;
1594                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
1595                 if (dcb_conf->pfc_en)
1596                         eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT|ETH_DCB_PFC_SUPPORT;
1597                 else
1598                         eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
1599
1600                 (void)(rte_memcpy(&eth_conf->rx_adv_conf.dcb_rx_conf, &rx_conf,
1601                                 sizeof(struct rte_eth_dcb_rx_conf)));
1602                 (void)(rte_memcpy(&eth_conf->tx_adv_conf.dcb_tx_conf, &tx_conf,
1603                                 sizeof(struct rte_eth_dcb_tx_conf)));
1604         }
1605
1606         return 0;
1607 }
1608
1609 int
1610 init_port_dcb_config(portid_t pid,struct dcb_config *dcb_conf)
1611 {
1612         struct rte_eth_conf port_conf;
1613         struct rte_port *rte_port;
1614         int retval;
1615         uint16_t nb_vlan;
1616         uint16_t i;
1617
1618         /* rxq and txq configuration in dcb mode */
1619         nb_rxq = 128;
1620         nb_txq = 128;
1621         rx_free_thresh = 64;
1622
1623         memset(&port_conf,0,sizeof(struct rte_eth_conf));
1624         /* Enter DCB configuration status */
1625         dcb_config = 1;
1626
1627         nb_vlan = sizeof( vlan_tags )/sizeof( vlan_tags[ 0 ]);
1628         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
1629         retval = get_eth_dcb_conf(&port_conf, dcb_conf);
1630         if (retval < 0)
1631                 return retval;
1632
1633         rte_port = &ports[pid];
1634         memcpy(&rte_port->dev_conf, &port_conf,sizeof(struct rte_eth_conf));
1635
1636         rte_port->rx_conf.rx_thresh = rx_thresh;
1637         rte_port->rx_conf.rx_free_thresh = rx_free_thresh;
1638         rte_port->tx_conf.tx_thresh = tx_thresh;
1639         rte_port->tx_conf.tx_rs_thresh = tx_rs_thresh;
1640         rte_port->tx_conf.tx_free_thresh = tx_free_thresh;
1641         /* VLAN filter */
1642         rte_port->dev_conf.rxmode.hw_vlan_filter = 1;
1643         for (i = 0; i < nb_vlan; i++){
1644                 rx_vft_set(pid, vlan_tags[i], 1);
1645         }
1646
1647         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
1648         map_port_queue_stats_mapping_registers(pid, rte_port);
1649
1650         return 0;
1651 }
1652
1653 #ifdef RTE_EXEC_ENV_BAREMETAL
1654 #define main _main
1655 #endif
1656
1657 int
1658 main(int argc, char** argv)
1659 {
1660         int  diag;
1661         uint8_t port_id;
1662
1663         diag = rte_eal_init(argc, argv);
1664         if (diag < 0)
1665                 rte_panic("Cannot init EAL\n");
1666
1667         if (rte_pmd_init_all())
1668                 rte_panic("Cannot init PMD\n");
1669
1670         if (rte_eal_pci_probe())
1671                 rte_panic("Cannot probe PCI\n");
1672
1673         nb_ports = (portid_t) rte_eth_dev_count();
1674         if (nb_ports == 0)
1675                 rte_exit(EXIT_FAILURE, "No probed ethernet devices - "
1676                                                         "check that "
1677                           "CONFIG_RTE_LIBRTE_IGB_PMD=y and that "
1678                           "CONFIG_RTE_LIBRTE_EM_PMD=y and that "
1679                           "CONFIG_RTE_LIBRTE_IXGBE_PMD=y in your "
1680                           "configuration file\n");
1681
1682         set_def_fwd_config();
1683         if (nb_lcores == 0)
1684                 rte_panic("Empty set of forwarding logical cores - check the "
1685                           "core mask supplied in the command parameters\n");
1686
1687         argc -= diag;
1688         argv += diag;
1689         if (argc > 1)
1690                 launch_args_parse(argc, argv);
1691
1692         if (nb_rxq > nb_txq)
1693                 printf("Warning: nb_rxq=%d enables RSS configuration, "
1694                        "but nb_txq=%d will prevent to fully test it.\n",
1695                        nb_rxq, nb_txq);
1696
1697         init_config();
1698         start_port(RTE_PORT_ALL);
1699
1700         /* set all ports to promiscuous mode by default */
1701         for (port_id = 0; port_id < nb_ports; port_id++)
1702                 rte_eth_promiscuous_enable(port_id);
1703
1704         if (interactive == 1)
1705                 prompt();
1706         else {
1707                 char c;
1708                 int rc;
1709
1710                 printf("No commandline core given, start packet forwarding\n");
1711                 start_packet_forwarding(0);
1712                 printf("Press enter to exit\n");
1713                 rc = read(0, &c, 1);
1714                 if (rc < 0)
1715                         return 1;
1716         }
1717
1718         return 0;
1719 }