app/test-pmd/testpmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdarg.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <signal.h>
  38 #include <string.h>
  39 #include <time.h>
  40 #include <fcntl.h>
  41 #include <sys/types.h>
  42 #include <errno.h>
  43
  44 #include <sys/queue.h>
  45 #include <sys/stat.h>
  46
  47 #include <stdint.h>
  48 #include <unistd.h>
  49 #include <inttypes.h>
  50
  51 #include <rte_common.h>
  52 #include <rte_byteorder.h>
  53 #include <rte_log.h>
  54 #include <rte_debug.h>
  55 #include <rte_cycles.h>
  56 #include <rte_memory.h>
  57 #include <rte_memcpy.h>
  58 #include <rte_memzone.h>
  59 #include <rte_launch.h>
  60 #include <rte_tailq.h>
  61 #include <rte_eal.h>
  62 #include <rte_per_lcore.h>
  63 #include <rte_lcore.h>
  64 #include <rte_atomic.h>
  65 #include <rte_branch_prediction.h>
  66 #include <rte_ring.h>
  67 #include <rte_mempool.h>
  68 #include <rte_malloc.h>
  69 #include <rte_mbuf.h>
  70 #include <rte_interrupts.h>
  71 #include <rte_pci.h>
  72 #include <rte_ether.h>
  73 #include <rte_ethdev.h>
  74 #include <rte_string_fns.h>
  75 #ifdef RTE_LIBRTE_PMD_XENVIRT
  76 #include <rte_eth_xenvirt.h>
  77 #endif
  78
  79 #include "testpmd.h"
  80 #include "mempool_osdep.h"
  81
  82 uint16_t verbose_level = 0; /**< Silent by default. */
  83
  84 /* use master core for command line ? */
  85 uint8_t interactive = 0;
  86 uint8_t auto_start = 0;
  87
  88 /*
  89  * NUMA support configuration.
  90  * When set, the NUMA support attempts to dispatch the allocation of the
  91  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  92  * probed ports among the CPU sockets 0 and 1.
  93  * Otherwise, all memory is allocated from CPU socket 0.
  94  */
  95 uint8_t numa_support = 0; /**< No numa support by default */
  96
  97 /*
  98  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
  99  * not configured.
 100  */
 101 uint8_t socket_num = UMA_NO_CONFIG;
 102
 103 /*
 104  * Use ANONYMOUS mapped memory (might be not physically continuous) for mbufs.
 105  */
 106 uint8_t mp_anon = 0;
 107
 108 /*
 109  * Record the Ethernet address of peer target ports to which packets are
 110  * forwarded.
 111  * Must be instanciated with the ethernet addresses of peer traffic generator
 112  * ports.
 113  */
 114 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 115 portid_t nb_peer_eth_addrs = 0;
 116
 117 /*
 118  * Probed Target Environment.
 119  */
 120 struct rte_port *ports;        /**< For all probed ethernet ports. */
 121 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 122 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 123 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 124
 125 /*
 126  * Test Forwarding Configuration.
 127  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 128  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 129  */
 130 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 131 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 132 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 133 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 134
 135 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 136 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 137
 138 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 139 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 140
 141 /*
 142  * Forwarding engines.
 143  */
 144 struct fwd_engine * fwd_engines[] = {
 145         &io_fwd_engine,
 146         &mac_fwd_engine,
 147         &mac_retry_fwd_engine,
 148         &mac_swap_engine,
 149         &flow_gen_engine,
 150         &rx_only_engine,
 151         &tx_only_engine,
 152         &csum_fwd_engine,
 153         &icmp_echo_engine,
 154 #ifdef RTE_LIBRTE_IEEE1588
 155         &ieee1588_fwd_engine,
 156 #endif
 157         NULL,
 158 };
 159
 160 struct fwd_config cur_fwd_config;
 161 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 162
 163 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 164 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 165                                       * specified on command-line. */
 166
 167 /*
 168  * Configuration of packet segments used by the "txonly" processing engine.
 169  */
 170 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 171 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 172         TXONLY_DEF_PACKET_LEN,
 173 };
 174 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 175
 176 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 177 uint16_t mb_mempool_cache = DEF_PKT_BURST; /**< Size of mbuf mempool cache. */
 178
 179 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 180 uint8_t dcb_config = 0;
 181
 182 /* Whether the dcb is in testing status */
 183 uint8_t dcb_test = 0;
 184
 185 /* DCB on and VT on mapping is default */
 186 enum dcb_queue_mapping_mode dcb_q_mapping = DCB_VT_Q_MAPPING;
 187
 188 /*
 189  * Configurable number of RX/TX queues.
 190  */
 191 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 192 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 193
 194 /*
 195  * Configurable number of RX/TX ring descriptors.
 196  */
 197 #define RTE_TEST_RX_DESC_DEFAULT 128
 198 #define RTE_TEST_TX_DESC_DEFAULT 512
 199 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 200 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 201
 202 /*
 203  * Configurable values of RX and TX ring threshold registers.
 204  */
 205 #define RX_PTHRESH 8 /**< Default value of RX prefetch threshold register. */
 206 #define RX_HTHRESH 8 /**< Default value of RX host threshold register. */
 207 #define RX_WTHRESH 4 /**< Default value of RX write-back threshold register. */
 208
 209 #define TX_PTHRESH 36 /**< Default value of TX prefetch threshold register. */
 210 #define TX_HTHRESH 0 /**< Default value of TX host threshold register. */
 211 #define TX_WTHRESH 0 /**< Default value of TX write-back threshold register. */
 212
 213 struct rte_eth_thresh rx_thresh = {
 214         .pthresh = RX_PTHRESH,
 215         .hthresh = RX_HTHRESH,
 216         .wthresh = RX_WTHRESH,
 217 };
 218
 219 struct rte_eth_thresh tx_thresh = {
 220         .pthresh = TX_PTHRESH,
 221         .hthresh = TX_HTHRESH,
 222         .wthresh = TX_WTHRESH,
 223 };
 224
 225 /*
 226  * Configurable value of RX free threshold.
 227  */
 228 uint16_t rx_free_thresh = 0; /* Immediately free RX descriptors by default. */
 229
 230 /*
 231  * Configurable value of RX drop enable.
 232  */
 233 uint8_t rx_drop_en = 0; /* Drop packets when no descriptors for queue. */
 234
 235 /*
 236  * Configurable value of TX free threshold.
 237  */
 238 uint16_t tx_free_thresh = 0; /* Use default values. */
 239
 240 /*
 241  * Configurable value of TX RS bit threshold.
 242  */
 243 uint16_t tx_rs_thresh = 0; /* Use default values. */
 244
 245 /*
 246  * Configurable value of TX queue flags.
 247  */
 248 uint32_t txq_flags = 0; /* No flags set. */
 249
 250 /*
 251  * Receive Side Scaling (RSS) configuration.
 252  */
 253 uint16_t rss_hf = ETH_RSS_IPV4 | ETH_RSS_IPV6; /* RSS IP by default. */
 254
 255 /*
 256  * Port topology configuration
 257  */
 258 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 259
 260 /*
 261  * Avoids to flush all the RX streams before starts forwarding.
 262  */
 263 uint8_t no_flush_rx = 0; /* flush by default */
 264
 265 /*
 266  * NIC bypass mode configuration options.
 267  */
 268 #ifdef RTE_NIC_BYPASS
 269
 270 /* The NIC bypass watchdog timeout. */
 271 uint32_t bypass_timeout = RTE_BYPASS_TMT_OFF;
 272
 273 #endif
 274
 275 /*
 276  * Ethernet device configuration.
 277  */
 278 struct rte_eth_rxmode rx_mode = {
 279         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 280         .split_hdr_size = 0,
 281         .header_split   = 0, /**< Header Split disabled. */
 282         .hw_ip_checksum = 0, /**< IP checksum offload disabled. */
 283         .hw_vlan_filter = 1, /**< VLAN filtering enabled. */
 284         .hw_vlan_strip  = 1, /**< VLAN strip enabled. */
 285         .hw_vlan_extend = 0, /**< Extended VLAN disabled. */
 286         .jumbo_frame    = 0, /**< Jumbo Frame Support disabled. */
 287         .hw_strip_crc   = 0, /**< CRC stripping by hardware disabled. */
 288 };
 289
 290 struct rte_fdir_conf fdir_conf = {
 291         .mode = RTE_FDIR_MODE_NONE,
 292         .pballoc = RTE_FDIR_PBALLOC_64K,
 293         .status = RTE_FDIR_REPORT_STATUS,
 294         .flexbytes_offset = 0x6,
 295         .drop_queue = 127,
 296 };
 297
 298 static volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 299
 300 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 301 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 302
 303 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 304 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 305
 306 uint16_t nb_tx_queue_stats_mappings = 0;
 307 uint16_t nb_rx_queue_stats_mappings = 0;
 308
 309 /* Forward function declarations */
 310 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port);
 311 static void check_all_ports_link_status(uint8_t port_num, uint32_t port_mask);
 312
 313 /*
 314  * Check if all the ports are started.
 315  * If yes, return positive value. If not, return zero.
 316  */
 317 static int all_ports_started(void);
 318
 319 /*
 320  * Setup default configuration.
 321  */
 322 static void
 323 set_default_fwd_lcores_config(void)
 324 {
 325         unsigned int i;
 326         unsigned int nb_lc;
 327
 328         nb_lc = 0;
 329         for (i = 0; i < RTE_MAX_LCORE; i++) {
 330                 if (! rte_lcore_is_enabled(i))
 331                         continue;
 332                 if (i == rte_get_master_lcore())
 333                         continue;
 334                 fwd_lcores_cpuids[nb_lc++] = i;
 335         }
 336         nb_lcores = (lcoreid_t) nb_lc;
 337         nb_cfg_lcores = nb_lcores;
 338         nb_fwd_lcores = 1;
 339 }
 340
 341 static void
 342 set_def_peer_eth_addrs(void)
 343 {
 344         portid_t i;
 345
 346         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 347                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 348                 peer_eth_addrs[i].addr_bytes[5] = i;
 349         }
 350 }
 351
 352 static void
 353 set_default_fwd_ports_config(void)
 354 {
 355         portid_t pt_id;
 356
 357         for (pt_id = 0; pt_id < nb_ports; pt_id++)
 358                 fwd_ports_ids[pt_id] = pt_id;
 359
 360         nb_cfg_ports = nb_ports;
 361         nb_fwd_ports = nb_ports;
 362 }
 363
 364 void
 365 set_def_fwd_config(void)
 366 {
 367         set_default_fwd_lcores_config();
 368         set_def_peer_eth_addrs();
 369         set_default_fwd_ports_config();
 370 }
 371
 372 /*
 373  * Configuration initialisation done once at init time.
 374  */
 375 struct mbuf_ctor_arg {
 376         uint16_t seg_buf_offset; /**< offset of data in data segment of mbuf. */
 377         uint16_t seg_buf_size;   /**< size of data segment in mbuf. */
 378 };
 379
 380 struct mbuf_pool_ctor_arg {
 381         uint16_t seg_buf_size; /**< size of data segment in mbuf. */
 382 };
 383
 384 static void
 385 testpmd_mbuf_ctor(struct rte_mempool *mp,
 386                   void *opaque_arg,
 387                   void *raw_mbuf,
 388                   __attribute__((unused)) unsigned i)
 389 {
 390         struct mbuf_ctor_arg *mb_ctor_arg;
 391         struct rte_mbuf    *mb;
 392
 393         mb_ctor_arg = (struct mbuf_ctor_arg *) opaque_arg;
 394         mb = (struct rte_mbuf *) raw_mbuf;
 395
 396         mb->type         = RTE_MBUF_PKT;
 397         mb->pool         = mp;
 398         mb->buf_addr     = (void *) ((char *)mb + mb_ctor_arg->seg_buf_offset);
 399         mb->buf_physaddr = (uint64_t) (rte_mempool_virt2phy(mp, mb) +
 400                         mb_ctor_arg->seg_buf_offset);
 401         mb->buf_len      = mb_ctor_arg->seg_buf_size;
 402         mb->type         = RTE_MBUF_PKT;
 403         mb->ol_flags     = 0;
 404         mb->pkt.data     = (char *) mb->buf_addr + RTE_PKTMBUF_HEADROOM;
 405         mb->pkt.nb_segs  = 1;
 406         mb->pkt.vlan_macip.data = 0;
 407         mb->pkt.hash.rss = 0;
 408 }
 409
 410 static void
 411 testpmd_mbuf_pool_ctor(struct rte_mempool *mp,
 412                        void *opaque_arg)
 413 {
 414         struct mbuf_pool_ctor_arg      *mbp_ctor_arg;
 415         struct rte_pktmbuf_pool_private *mbp_priv;
 416
 417         if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
 418                 printf("%s(%s) private_data_size %d < %d\n",
 419                        __func__, mp->name, (int) mp->private_data_size,
 420                        (int) sizeof(struct rte_pktmbuf_pool_private));
 421                 return;
 422         }
 423         mbp_ctor_arg = (struct mbuf_pool_ctor_arg *) opaque_arg;
 424         mbp_priv = rte_mempool_get_priv(mp);
 425         mbp_priv->mbuf_data_room_size = mbp_ctor_arg->seg_buf_size;
 426 }
 427
 428 static void
 429 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 430                  unsigned int socket_id)
 431 {
 432         char pool_name[RTE_MEMPOOL_NAMESIZE];
 433         struct rte_mempool *rte_mp;
 434         struct mbuf_pool_ctor_arg mbp_ctor_arg;
 435         struct mbuf_ctor_arg mb_ctor_arg;
 436         uint32_t mb_size;
 437
 438         mbp_ctor_arg.seg_buf_size = (uint16_t) (RTE_PKTMBUF_HEADROOM +
 439                                                 mbuf_seg_size);
 440         mb_ctor_arg.seg_buf_offset =
 441                 (uint16_t) CACHE_LINE_ROUNDUP(sizeof(struct rte_mbuf));
 442         mb_ctor_arg.seg_buf_size = mbp_ctor_arg.seg_buf_size;
 443         mb_size = mb_ctor_arg.seg_buf_offset + mb_ctor_arg.seg_buf_size;
 444         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 445
 446 #ifdef RTE_LIBRTE_PMD_XENVIRT
 447         rte_mp = rte_mempool_gntalloc_create(pool_name, nb_mbuf, mb_size,
 448                                    (unsigned) mb_mempool_cache,
 449                                    sizeof(struct rte_pktmbuf_pool_private),
 450                                    testpmd_mbuf_pool_ctor, &mbp_ctor_arg,
 451                                    testpmd_mbuf_ctor, &mb_ctor_arg,
 452                                    socket_id, 0);
 453
 454
 455
 456 #else
 457         if (mp_anon != 0)
 458                 rte_mp = mempool_anon_create(pool_name, nb_mbuf, mb_size,
 459                                     (unsigned) mb_mempool_cache,
 460                                     sizeof(struct rte_pktmbuf_pool_private),
 461                                     testpmd_mbuf_pool_ctor, &mbp_ctor_arg,
 462                                     testpmd_mbuf_ctor, &mb_ctor_arg,
 463                                     socket_id, 0);
 464         else
 465                 rte_mp = rte_mempool_create(pool_name, nb_mbuf, mb_size,
 466                                     (unsigned) mb_mempool_cache,
 467                                     sizeof(struct rte_pktmbuf_pool_private),
 468                                     testpmd_mbuf_pool_ctor, &mbp_ctor_arg,
 469                                     testpmd_mbuf_ctor, &mb_ctor_arg,
 470                                     socket_id, 0);
 471
 472 #endif
 473
 474         if (rte_mp == NULL) {
 475                 rte_exit(EXIT_FAILURE, "Creation of mbuf pool for socket %u "
 476                                                 "failed\n", socket_id);
 477         } else if (verbose_level > 0) {
 478                 rte_mempool_dump(rte_mp);
 479         }
 480 }
 481
 482 /*
 483  * Check given socket id is valid or not with NUMA mode,
 484  * if valid, return 0, else return -1
 485  */
 486 static int
 487 check_socket_id(const unsigned int socket_id)
 488 {
 489         static int warning_once = 0;
 490
 491         if (socket_id >= MAX_SOCKET) {
 492                 if (!warning_once && numa_support)
 493                         printf("Warning: NUMA should be configured manually by"
 494                                " using --port-numa-config and"
 495                                " --ring-numa-config parameters along with"
 496                                " --numa.\n");
 497                 warning_once = 1;
 498                 return -1;
 499         }
 500         return 0;
 501 }
 502
 503 static void
 504 init_config(void)
 505 {
 506         portid_t pid;
 507         struct rte_port *port;
 508         struct rte_mempool *mbp;
 509         unsigned int nb_mbuf_per_pool;
 510         lcoreid_t  lc_id;
 511         uint8_t port_per_socket[MAX_SOCKET];
 512
 513         memset(port_per_socket,0,MAX_SOCKET);
 514         /* Configuration of logical cores. */
 515         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
 516                                 sizeof(struct fwd_lcore *) * nb_lcores,
 517                                 CACHE_LINE_SIZE);
 518         if (fwd_lcores == NULL) {
 519                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
 520                                                         "failed\n", nb_lcores);
 521         }
 522         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 523                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
 524                                                sizeof(struct fwd_lcore),
 525                                                CACHE_LINE_SIZE);
 526                 if (fwd_lcores[lc_id] == NULL) {
 527                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
 528                                                                 "failed\n");
 529                 }
 530                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
 531         }
 532
 533         /*
 534          * Create pools of mbuf.
 535          * If NUMA support is disabled, create a single pool of mbuf in
 536          * socket 0 memory by default.
 537          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
 538          *
 539          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
 540          * nb_txd can be configured at run time.
 541          */
 542         if (param_total_num_mbufs)
 543                 nb_mbuf_per_pool = param_total_num_mbufs;
 544         else {
 545                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + (nb_lcores * mb_mempool_cache)
 546                                 + RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
 547
 548                 if (!numa_support)
 549                         nb_mbuf_per_pool = (nb_mbuf_per_pool * nb_ports);
 550         }
 551
 552         if (!numa_support) {
 553                 if (socket_num == UMA_NO_CONFIG)
 554                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0);
 555                 else
 556                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
 557                                                  socket_num);
 558         }
 559
 560         /* Configuration of Ethernet ports. */
 561         ports = rte_zmalloc("testpmd: ports",
 562                             sizeof(struct rte_port) * nb_ports,
 563                             CACHE_LINE_SIZE);
 564         if (ports == NULL) {
 565                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d struct rte_port) "
 566                                                         "failed\n", nb_ports);
 567         }
 568
 569         for (pid = 0; pid < nb_ports; pid++) {
 570                 port = &ports[pid];
 571                 rte_eth_dev_info_get(pid, &port->dev_info);
 572
 573                 if (numa_support) {
 574                         if (port_numa[pid] != NUMA_NO_CONFIG)
 575                                 port_per_socket[port_numa[pid]]++;
 576                         else {
 577                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
 578
 579                                 /* if socket_id is invalid, set to 0 */
 580                                 if (check_socket_id(socket_id) < 0)
 581                                         socket_id = 0;
 582                                 port_per_socket[socket_id]++;
 583                         }
 584                 }
 585
 586                 /* set flag to initialize port/queue */
 587                 port->need_reconfig = 1;
 588                 port->need_reconfig_queues = 1;
 589         }
 590
 591         if (numa_support) {
 592                 uint8_t i;
 593                 unsigned int nb_mbuf;
 594
 595                 if (param_total_num_mbufs)
 596                         nb_mbuf_per_pool = nb_mbuf_per_pool/nb_ports;
 597
 598                 for (i = 0; i < MAX_SOCKET; i++) {
 599                         nb_mbuf = (nb_mbuf_per_pool *
 600                                                 port_per_socket[i]);
 601                         if (nb_mbuf)
 602                                 mbuf_pool_create(mbuf_data_size,
 603                                                 nb_mbuf,i);
 604                 }
 605         }
 606         init_port_config();
 607
 608         /*
 609          * Records which Mbuf pool to use by each logical core, if needed.
 610          */
 611         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 612                 mbp = mbuf_pool_find(rte_lcore_to_socket_id(lc_id));
 613                 if (mbp == NULL)
 614                         mbp = mbuf_pool_find(0);
 615                 fwd_lcores[lc_id]->mbp = mbp;
 616         }
 617
 618         /* Configuration of packet forwarding streams. */
 619         if (init_fwd_streams() < 0)
 620                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 621 }
 622
 623 int
 624 init_fwd_streams(void)
 625 {
 626         portid_t pid;
 627         struct rte_port *port;
 628         streamid_t sm_id, nb_fwd_streams_new;
 629
 630         /* set socket id according to numa or not */
 631         for (pid = 0; pid < nb_ports; pid++) {
 632                 port = &ports[pid];
 633                 if (nb_rxq > port->dev_info.max_rx_queues) {
 634                         printf("Fail: nb_rxq(%d) is greater than "
 635                                 "max_rx_queues(%d)\n", nb_rxq,
 636                                 port->dev_info.max_rx_queues);
 637                         return -1;
 638                 }
 639                 if (nb_txq > port->dev_info.max_tx_queues) {
 640                         printf("Fail: nb_txq(%d) is greater than "
 641                                 "max_tx_queues(%d)\n", nb_txq,
 642                                 port->dev_info.max_tx_queues);
 643                         return -1;
 644                 }
 645                 if (numa_support) {
 646                         if (port_numa[pid] != NUMA_NO_CONFIG)
 647                                 port->socket_id = port_numa[pid];
 648                         else {
 649                                 port->socket_id = rte_eth_dev_socket_id(pid);
 650
 651                                 /* if socket_id is invalid, set to 0 */
 652                                 if (check_socket_id(port->socket_id) < 0)
 653                                         port->socket_id = 0;
 654                         }
 655                 }
 656                 else {
 657                         if (socket_num == UMA_NO_CONFIG)
 658                                 port->socket_id = 0;
 659                         else
 660                                 port->socket_id = socket_num;
 661                 }
 662         }
 663
 664         nb_fwd_streams_new = (streamid_t)(nb_ports * nb_rxq);
 665         if (nb_fwd_streams_new == nb_fwd_streams)
 666                 return 0;
 667         /* clear the old */
 668         if (fwd_streams != NULL) {
 669                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 670                         if (fwd_streams[sm_id] == NULL)
 671                                 continue;
 672                         rte_free(fwd_streams[sm_id]);
 673                         fwd_streams[sm_id] = NULL;
 674                 }
 675                 rte_free(fwd_streams);
 676                 fwd_streams = NULL;
 677         }
 678
 679         /* init new */
 680         nb_fwd_streams = nb_fwd_streams_new;
 681         fwd_streams = rte_zmalloc("testpmd: fwd_streams",
 682                 sizeof(struct fwd_stream *) * nb_fwd_streams, CACHE_LINE_SIZE);
 683         if (fwd_streams == NULL)
 684                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_stream *)) "
 685                                                 "failed\n", nb_fwd_streams);
 686
 687         for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 688                 fwd_streams[sm_id] = rte_zmalloc("testpmd: struct fwd_stream",
 689                                 sizeof(struct fwd_stream), CACHE_LINE_SIZE);
 690                 if (fwd_streams[sm_id] == NULL)
 691                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_stream)"
 692                                                                 " failed\n");
 693         }
 694
 695         return 0;
 696 }
 697
 698 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 699 static void
 700 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
 701 {
 702         unsigned int total_burst;
 703         unsigned int nb_burst;
 704         unsigned int burst_stats[3];
 705         uint16_t pktnb_stats[3];
 706         uint16_t nb_pkt;
 707         int burst_percent[3];
 708
 709         /*
 710          * First compute the total number of packet bursts and the
 711          * two highest numbers of bursts of the same number of packets.
 712          */
 713         total_burst = 0;
 714         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
 715         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
 716         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
 717                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
 718                 if (nb_burst == 0)
 719                         continue;
 720                 total_burst += nb_burst;
 721                 if (nb_burst > burst_stats[0]) {
 722                         burst_stats[1] = burst_stats[0];
 723                         pktnb_stats[1] = pktnb_stats[0];
 724                         burst_stats[0] = nb_burst;
 725                         pktnb_stats[0] = nb_pkt;
 726                 }
 727         }
 728         if (total_burst == 0)
 729                 return;
 730         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
 731         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
 732                burst_percent[0], (int) pktnb_stats[0]);
 733         if (burst_stats[0] == total_burst) {
 734                 printf("]\n");
 735                 return;
 736         }
 737         if (burst_stats[0] + burst_stats[1] == total_burst) {
 738                 printf(" + %d%% of %d pkts]\n",
 739                        100 - burst_percent[0], pktnb_stats[1]);
 740                 return;
 741         }
 742         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
 743         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
 744         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
 745                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
 746                 return;
 747         }
 748         printf(" + %d%% of %d pkts + %d%% of others]\n",
 749                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
 750 }
 751 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
 752
 753 static void
 754 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats)
 755 {
 756         struct rte_port *port;
 757         uint8_t i;
 758
 759         static const char *fwd_stats_border = "----------------------";
 760
 761         port = &ports[port_id];
 762         printf("\n  %s Forward statistics for port %-2d %s\n",
 763                fwd_stats_border, port_id, fwd_stats_border);
 764
 765         if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) {
 766                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
 767                        "%-"PRIu64"\n",
 768                        stats->ipackets, stats->ierrors,
 769                        (uint64_t) (stats->ipackets + stats->ierrors));
 770
 771                 if (cur_fwd_eng == &csum_fwd_engine)
 772                         printf("  Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n",
 773                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 774
 775                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
 776                        "%-"PRIu64"\n",
 777                        stats->opackets, port->tx_dropped,
 778                        (uint64_t) (stats->opackets + port->tx_dropped));
 779
 780                 if (stats->rx_nombuf > 0)
 781                         printf("  RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf);
 782
 783         }
 784         else {
 785                 printf("  RX-packets:             %14"PRIu64"    RX-dropped:%14"PRIu64"    RX-total:"
 786                        "%14"PRIu64"\n",
 787                        stats->ipackets, stats->ierrors,
 788                        (uint64_t) (stats->ipackets + stats->ierrors));
 789
 790                 if (cur_fwd_eng == &csum_fwd_engine)
 791                         printf("  Bad-ipcsum:%14"PRIu64"    Bad-l4csum:%14"PRIu64"\n",
 792                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 793
 794                 printf("  TX-packets:             %14"PRIu64"    TX-dropped:%14"PRIu64"    TX-total:"
 795                        "%14"PRIu64"\n",
 796                        stats->opackets, port->tx_dropped,
 797                        (uint64_t) (stats->opackets + port->tx_dropped));
 798
 799                 if (stats->rx_nombuf > 0)
 800                         printf("  RX-nombufs:%14"PRIu64"\n", stats->rx_nombuf);
 801         }
 802
 803         /* Display statistics of XON/XOFF pause frames, if any. */
 804         if ((stats->tx_pause_xon  | stats->rx_pause_xon |
 805              stats->tx_pause_xoff | stats->rx_pause_xoff) > 0) {
 806                 printf("  RX-XOFF:    %-14"PRIu64" RX-XON:     %-14"PRIu64"\n",
 807                        stats->rx_pause_xoff, stats->rx_pause_xon);
 808                 printf("  TX-XOFF:    %-14"PRIu64" TX-XON:     %-14"PRIu64"\n",
 809                        stats->tx_pause_xoff, stats->tx_pause_xon);
 810         }
 811
 812 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 813         if (port->rx_stream)
 814                 pkt_burst_stats_display("RX",
 815                         &port->rx_stream->rx_burst_stats);
 816         if (port->tx_stream)
 817                 pkt_burst_stats_display("TX",
 818                         &port->tx_stream->tx_burst_stats);
 819 #endif
 820         /* stats fdir */
 821         if (fdir_conf.mode != RTE_FDIR_MODE_NONE)
 822                 printf("  Fdirmiss:%14"PRIu64"    Fdirmatch:%14"PRIu64"\n",
 823                        stats->fdirmiss,
 824                        stats->fdirmatch);
 825
 826         if (port->rx_queue_stats_mapping_enabled) {
 827                 printf("\n");
 828                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 829                         printf("  Stats reg %2d RX-packets:%14"PRIu64
 830                                "     RX-errors:%14"PRIu64
 831                                "    RX-bytes:%14"PRIu64"\n",
 832                                i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]);
 833                 }
 834                 printf("\n");
 835         }
 836         if (port->tx_queue_stats_mapping_enabled) {
 837                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 838                         printf("  Stats reg %2d TX-packets:%14"PRIu64
 839                                "                                 TX-bytes:%14"PRIu64"\n",
 840                                i, stats->q_opackets[i], stats->q_obytes[i]);
 841                 }
 842         }
 843
 844         printf("  %s--------------------------------%s\n",
 845                fwd_stats_border, fwd_stats_border);
 846 }
 847
 848 static void
 849 fwd_stream_stats_display(streamid_t stream_id)
 850 {
 851         struct fwd_stream *fs;
 852         static const char *fwd_top_stats_border = "-------";
 853
 854         fs = fwd_streams[stream_id];
 855         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
 856             (fs->fwd_dropped == 0))
 857                 return;
 858         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
 859                "TX Port=%2d/Queue=%2d %s\n",
 860                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
 861                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
 862         printf("  RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u",
 863                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
 864
 865         /* if checksum mode */
 866         if (cur_fwd_eng == &csum_fwd_engine) {
 867                printf("  RX- bad IP checksum: %-14u  Rx- bad L4 checksum: "
 868                         "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum);
 869         }
 870
 871 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 872         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
 873         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
 874 #endif
 875 }
 876
 877 static void
 878 flush_fwd_rx_queues(void)
 879 {
 880         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 881         portid_t  rxp;
 882         portid_t port_id;
 883         queueid_t rxq;
 884         uint16_t  nb_rx;
 885         uint16_t  i;
 886         uint8_t   j;
 887
 888         for (j = 0; j < 2; j++) {
 889                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
 890                         for (rxq = 0; rxq < nb_rxq; rxq++) {
 891                                 port_id = fwd_ports_ids[rxp];
 892                                 do {
 893                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
 894                                                 pkts_burst, MAX_PKT_BURST);
 895                                         for (i = 0; i < nb_rx; i++)
 896                                                 rte_pktmbuf_free(pkts_burst[i]);
 897                                 } while (nb_rx > 0);
 898                         }
 899                 }
 900                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
 901         }
 902 }
 903
 904 static void
 905 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
 906 {
 907         struct fwd_stream **fsm;
 908         streamid_t nb_fs;
 909         streamid_t sm_id;
 910
 911         fsm = &fwd_streams[fc->stream_idx];
 912         nb_fs = fc->stream_nb;
 913         do {
 914                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
 915                         (*pkt_fwd)(fsm[sm_id]);
 916         } while (! fc->stopped);
 917 }
 918
 919 static int
 920 start_pkt_forward_on_core(void *fwd_arg)
 921 {
 922         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
 923                              cur_fwd_config.fwd_eng->packet_fwd);
 924         return 0;
 925 }
 926
 927 /*
 928  * Run the TXONLY packet forwarding engine to send a single burst of packets.
 929  * Used to start communication flows in network loopback test configurations.
 930  */
 931 static int
 932 run_one_txonly_burst_on_core(void *fwd_arg)
 933 {
 934         struct fwd_lcore *fwd_lc;
 935         struct fwd_lcore tmp_lcore;
 936
 937         fwd_lc = (struct fwd_lcore *) fwd_arg;
 938         tmp_lcore = *fwd_lc;
 939         tmp_lcore.stopped = 1;
 940         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
 941         return 0;
 942 }
 943
 944 /*
 945  * Launch packet forwarding:
 946  *     - Setup per-port forwarding context.
 947  *     - launch logical cores with their forwarding configuration.
 948  */
 949 static void
 950 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
 951 {
 952         port_fwd_begin_t port_fwd_begin;
 953         unsigned int i;
 954         unsigned int lc_id;
 955         int diag;
 956
 957         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
 958         if (port_fwd_begin != NULL) {
 959                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
 960                         (*port_fwd_begin)(fwd_ports_ids[i]);
 961         }
 962         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
 963                 lc_id = fwd_lcores_cpuids[i];
 964                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
 965                         fwd_lcores[i]->stopped = 0;
 966                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
 967                                                      fwd_lcores[i], lc_id);
 968                         if (diag != 0)
 969                                 printf("launch lcore %u failed - diag=%d\n",
 970                                        lc_id, diag);
 971                 }
 972         }
 973 }
 974
 975 /*
 976  * Launch packet forwarding configuration.
 977  */
 978 void
 979 start_packet_forwarding(int with_tx_first)
 980 {
 981         port_fwd_begin_t port_fwd_begin;
 982         port_fwd_end_t  port_fwd_end;
 983         struct rte_port *port;
 984         unsigned int i;
 985         portid_t   pt_id;
 986         streamid_t sm_id;
 987
 988         if (all_ports_started() == 0) {
 989                 printf("Not all ports were started\n");
 990                 return;
 991         }
 992         if (test_done == 0) {
 993                 printf("Packet forwarding already started\n");
 994                 return;
 995         }
 996         if(dcb_test) {
 997                 for (i = 0; i < nb_fwd_ports; i++) {
 998                         pt_id = fwd_ports_ids[i];
 999                         port = &ports[pt_id];
1000                         if (!port->dcb_flag) {
1001                                 printf("In DCB mode, all forwarding ports must "
1002                                        "be configured in this mode.\n");
1003                                 return;
1004                         }
1005                 }
1006                 if (nb_fwd_lcores == 1) {
1007                         printf("In DCB mode,the nb forwarding cores "
1008                                "should be larger than 1.\n");
1009                         return;
1010                 }
1011         }
1012         test_done = 0;
1013
1014         if(!no_flush_rx)
1015                 flush_fwd_rx_queues();
1016
1017         fwd_config_setup();
1018         rxtx_config_display();
1019
1020         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1021                 pt_id = fwd_ports_ids[i];
1022                 port = &ports[pt_id];
1023                 rte_eth_stats_get(pt_id, &port->stats);
1024                 port->tx_dropped = 0;
1025
1026                 map_port_queue_stats_mapping_registers(pt_id, port);
1027         }
1028         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1029                 fwd_streams[sm_id]->rx_packets = 0;
1030                 fwd_streams[sm_id]->tx_packets = 0;
1031                 fwd_streams[sm_id]->fwd_dropped = 0;
1032                 fwd_streams[sm_id]->rx_bad_ip_csum = 0;
1033                 fwd_streams[sm_id]->rx_bad_l4_csum = 0;
1034
1035 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1036                 memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
1037                        sizeof(fwd_streams[sm_id]->rx_burst_stats));
1038                 memset(&fwd_streams[sm_id]->tx_burst_stats, 0,
1039                        sizeof(fwd_streams[sm_id]->tx_burst_stats));
1040 #endif
1041 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1042                 fwd_streams[sm_id]->core_cycles = 0;
1043 #endif
1044         }
1045         if (with_tx_first) {
1046                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1047                 if (port_fwd_begin != NULL) {
1048                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1049                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1050                 }
1051                 launch_packet_forwarding(run_one_txonly_burst_on_core);
1052                 rte_eal_mp_wait_lcore();
1053                 port_fwd_end = tx_only_engine.port_fwd_end;
1054                 if (port_fwd_end != NULL) {
1055                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1056                                 (*port_fwd_end)(fwd_ports_ids[i]);
1057                 }
1058         }
1059         launch_packet_forwarding(start_pkt_forward_on_core);
1060 }
1061
1062 void
1063 stop_packet_forwarding(void)
1064 {
1065         struct rte_eth_stats stats;
1066         struct rte_port *port;
1067         port_fwd_end_t  port_fwd_end;
1068         int i;
1069         portid_t   pt_id;
1070         streamid_t sm_id;
1071         lcoreid_t  lc_id;
1072         uint64_t total_recv;
1073         uint64_t total_xmit;
1074         uint64_t total_rx_dropped;
1075         uint64_t total_tx_dropped;
1076         uint64_t total_rx_nombuf;
1077         uint64_t tx_dropped;
1078         uint64_t rx_bad_ip_csum;
1079         uint64_t rx_bad_l4_csum;
1080 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1081         uint64_t fwd_cycles;
1082 #endif
1083         static const char *acc_stats_border = "+++++++++++++++";
1084
1085         if (all_ports_started() == 0) {
1086                 printf("Not all ports were started\n");
1087                 return;
1088         }
1089         if (test_done) {
1090                 printf("Packet forwarding not started\n");
1091                 return;
1092         }
1093         printf("Telling cores to stop...");
1094         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1095                 fwd_lcores[lc_id]->stopped = 1;
1096         printf("\nWaiting for lcores to finish...\n");
1097         rte_eal_mp_wait_lcore();
1098         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1099         if (port_fwd_end != NULL) {
1100                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1101                         pt_id = fwd_ports_ids[i];
1102                         (*port_fwd_end)(pt_id);
1103                 }
1104         }
1105 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1106         fwd_cycles = 0;
1107 #endif
1108         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1109                 if (cur_fwd_config.nb_fwd_streams >
1110                     cur_fwd_config.nb_fwd_ports) {
1111                         fwd_stream_stats_display(sm_id);
1112                         ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL;
1113                         ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL;
1114                 } else {
1115                         ports[fwd_streams[sm_id]->tx_port].tx_stream =
1116                                 fwd_streams[sm_id];
1117                         ports[fwd_streams[sm_id]->rx_port].rx_stream =
1118                                 fwd_streams[sm_id];
1119                 }
1120                 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped;
1121                 tx_dropped = (uint64_t) (tx_dropped +
1122                                          fwd_streams[sm_id]->fwd_dropped);
1123                 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped;
1124
1125                 rx_bad_ip_csum =
1126                         ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum;
1127                 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum +
1128                                          fwd_streams[sm_id]->rx_bad_ip_csum);
1129                 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum =
1130                                                         rx_bad_ip_csum;
1131
1132                 rx_bad_l4_csum =
1133                         ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum;
1134                 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum +
1135                                          fwd_streams[sm_id]->rx_bad_l4_csum);
1136                 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum =
1137                                                         rx_bad_l4_csum;
1138
1139 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1140                 fwd_cycles = (uint64_t) (fwd_cycles +
1141                                          fwd_streams[sm_id]->core_cycles);
1142 #endif
1143         }
1144         total_recv = 0;
1145         total_xmit = 0;
1146         total_rx_dropped = 0;
1147         total_tx_dropped = 0;
1148         total_rx_nombuf  = 0;
1149         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1150                 pt_id = fwd_ports_ids[i];
1151
1152                 port = &ports[pt_id];
1153                 rte_eth_stats_get(pt_id, &stats);
1154                 stats.ipackets -= port->stats.ipackets;
1155                 port->stats.ipackets = 0;
1156                 stats.opackets -= port->stats.opackets;
1157                 port->stats.opackets = 0;
1158                 stats.ibytes   -= port->stats.ibytes;
1159                 port->stats.ibytes = 0;
1160                 stats.obytes   -= port->stats.obytes;
1161                 port->stats.obytes = 0;
1162                 stats.ierrors  -= port->stats.ierrors;
1163                 port->stats.ierrors = 0;
1164                 stats.oerrors  -= port->stats.oerrors;
1165                 port->stats.oerrors = 0;
1166                 stats.rx_nombuf -= port->stats.rx_nombuf;
1167                 port->stats.rx_nombuf = 0;
1168                 stats.fdirmatch -= port->stats.fdirmatch;
1169                 port->stats.rx_nombuf = 0;
1170                 stats.fdirmiss -= port->stats.fdirmiss;
1171                 port->stats.rx_nombuf = 0;
1172
1173                 total_recv += stats.ipackets;
1174                 total_xmit += stats.opackets;
1175                 total_rx_dropped += stats.ierrors;
1176                 total_tx_dropped += port->tx_dropped;
1177                 total_rx_nombuf  += stats.rx_nombuf;
1178
1179                 fwd_port_stats_display(pt_id, &stats);
1180         }
1181         printf("\n  %s Accumulated forward statistics for all ports"
1182                "%s\n",
1183                acc_stats_border, acc_stats_border);
1184         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1185                "%-"PRIu64"\n"
1186                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1187                "%-"PRIu64"\n",
1188                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1189                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1190         if (total_rx_nombuf > 0)
1191                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1192         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1193                "%s\n",
1194                acc_stats_border, acc_stats_border);
1195 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1196         if (total_recv > 0)
1197                 printf("\n  CPU cycles/packet=%u (total cycles="
1198                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1199                        (unsigned int)(fwd_cycles / total_recv),
1200                        fwd_cycles, total_recv);
1201 #endif
1202         printf("\nDone.\n");
1203         test_done = 1;
1204 }
1205
1206 static int
1207 all_ports_started(void)
1208 {
1209         portid_t pi;
1210         struct rte_port *port;
1211
1212         for (pi = 0; pi < nb_ports; pi++) {
1213                 port = &ports[pi];
1214                 /* Check if there is a port which is not started */
1215                 if (port->port_status != RTE_PORT_STARTED)
1216                         return 0;
1217         }
1218
1219         /* No port is not started */
1220         return 1;
1221 }
1222
1223 int
1224 start_port(portid_t pid)
1225 {
1226         int diag, need_check_link_status = 0;
1227         portid_t pi;
1228         queueid_t qi;
1229         struct rte_port *port;
1230
1231         if (test_done == 0) {
1232                 printf("Please stop forwarding first\n");
1233                 return -1;
1234         }
1235
1236         if (init_fwd_streams() < 0) {
1237                 printf("Fail from init_fwd_streams()\n");
1238                 return -1;
1239         }
1240
1241         if(dcb_config)
1242                 dcb_test = 1;
1243         for (pi = 0; pi < nb_ports; pi++) {
1244                 if (pid < nb_ports && pid != pi)
1245                         continue;
1246
1247                 port = &ports[pi];
1248                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1249                                                  RTE_PORT_HANDLING) == 0) {
1250                         printf("Port %d is now not stopped\n", pi);
1251                         continue;
1252                 }
1253
1254                 if (port->need_reconfig > 0) {
1255                         port->need_reconfig = 0;
1256
1257                         printf("Configuring Port %d (socket %u)\n", pi,
1258                                         port->socket_id);
1259                         /* configure port */
1260                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1261                                                 &(port->dev_conf));
1262                         if (diag != 0) {
1263                                 if (rte_atomic16_cmpset(&(port->port_status),
1264                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1265                                         printf("Port %d can not be set back "
1266                                                         "to stopped\n", pi);
1267                                 printf("Fail to configure port %d\n", pi);
1268                                 /* try to reconfigure port next time */
1269                                 port->need_reconfig = 1;
1270                                 return -1;
1271                         }
1272                 }
1273                 if (port->need_reconfig_queues > 0) {
1274                         port->need_reconfig_queues = 0;
1275                         /* setup tx queues */
1276                         for (qi = 0; qi < nb_txq; qi++) {
1277                                 if ((numa_support) &&
1278                                         (txring_numa[pi] != NUMA_NO_CONFIG))
1279                                         diag = rte_eth_tx_queue_setup(pi, qi,
1280                                                 nb_txd,txring_numa[pi],
1281                                                 &(port->tx_conf));
1282                                 else
1283                                         diag = rte_eth_tx_queue_setup(pi, qi,
1284                                                 nb_txd,port->socket_id,
1285                                                 &(port->tx_conf));
1286
1287                                 if (diag == 0)
1288                                         continue;
1289
1290                                 /* Fail to setup tx queue, return */
1291                                 if (rte_atomic16_cmpset(&(port->port_status),
1292                                                         RTE_PORT_HANDLING,
1293                                                         RTE_PORT_STOPPED) == 0)
1294                                         printf("Port %d can not be set back "
1295                                                         "to stopped\n", pi);
1296                                 printf("Fail to configure port %d tx queues\n", pi);
1297                                 /* try to reconfigure queues next time */
1298                                 port->need_reconfig_queues = 1;
1299                                 return -1;
1300                         }
1301                         /* setup rx queues */
1302                         for (qi = 0; qi < nb_rxq; qi++) {
1303                                 if ((numa_support) &&
1304                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
1305                                         struct rte_mempool * mp =
1306                                                 mbuf_pool_find(rxring_numa[pi]);
1307                                         if (mp == NULL) {
1308                                                 printf("Failed to setup RX queue:"
1309                                                         "No mempool allocation"
1310                                                         "on the socket %d\n",
1311                                                         rxring_numa[pi]);
1312                                                 return -1;
1313                                         }
1314
1315                                         diag = rte_eth_rx_queue_setup(pi, qi,
1316                                              nb_rxd,rxring_numa[pi],
1317                                              &(port->rx_conf),mp);
1318                                 }
1319                                 else
1320                                         diag = rte_eth_rx_queue_setup(pi, qi,
1321                                              nb_rxd,port->socket_id,
1322                                              &(port->rx_conf),
1323                                              mbuf_pool_find(port->socket_id));
1324
1325                                 if (diag == 0)
1326                                         continue;
1327
1328
1329                                 /* Fail to setup rx queue, return */
1330                                 if (rte_atomic16_cmpset(&(port->port_status),
1331                                                         RTE_PORT_HANDLING,
1332                                                         RTE_PORT_STOPPED) == 0)
1333                                         printf("Port %d can not be set back "
1334                                                         "to stopped\n", pi);
1335                                 printf("Fail to configure port %d rx queues\n", pi);
1336                                 /* try to reconfigure queues next time */
1337                                 port->need_reconfig_queues = 1;
1338                                 return -1;
1339                         }
1340                 }
1341                 /* start port */
1342                 if (rte_eth_dev_start(pi) < 0) {
1343                         printf("Fail to start port %d\n", pi);
1344
1345                         /* Fail to setup rx queue, return */
1346                         if (rte_atomic16_cmpset(&(port->port_status),
1347                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1348                                 printf("Port %d can not be set back to "
1349                                                         "stopped\n", pi);
1350                         continue;
1351                 }
1352
1353                 if (rte_atomic16_cmpset(&(port->port_status),
1354                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
1355                         printf("Port %d can not be set into started\n", pi);
1356
1357                 /* at least one port started, need checking link status */
1358                 need_check_link_status = 1;
1359         }
1360
1361         if (need_check_link_status)
1362                 check_all_ports_link_status(nb_ports, RTE_PORT_ALL);
1363         else
1364                 printf("Please stop the ports first\n");
1365
1366         printf("Done\n");
1367         return 0;
1368 }
1369
1370 void
1371 stop_port(portid_t pid)
1372 {
1373         portid_t pi;
1374         struct rte_port *port;
1375         int need_check_link_status = 0;
1376
1377         if (test_done == 0) {
1378                 printf("Please stop forwarding first\n");
1379                 return;
1380         }
1381         if (dcb_test) {
1382                 dcb_test = 0;
1383                 dcb_config = 0;
1384         }
1385         printf("Stopping ports...\n");
1386
1387         for (pi = 0; pi < nb_ports; pi++) {
1388                 if (pid < nb_ports && pid != pi)
1389                         continue;
1390
1391                 port = &ports[pi];
1392                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
1393                                                 RTE_PORT_HANDLING) == 0)
1394                         continue;
1395
1396                 rte_eth_dev_stop(pi);
1397
1398                 if (rte_atomic16_cmpset(&(port->port_status),
1399                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1400                         printf("Port %d can not be set into stopped\n", pi);
1401                 need_check_link_status = 1;
1402         }
1403         if (need_check_link_status)
1404                 check_all_ports_link_status(nb_ports, RTE_PORT_ALL);
1405
1406         printf("Done\n");
1407 }
1408
1409 void
1410 close_port(portid_t pid)
1411 {
1412         portid_t pi;
1413         struct rte_port *port;
1414
1415         if (test_done == 0) {
1416                 printf("Please stop forwarding first\n");
1417                 return;
1418         }
1419
1420         printf("Closing ports...\n");
1421
1422         for (pi = 0; pi < nb_ports; pi++) {
1423                 if (pid < nb_ports && pid != pi)
1424                         continue;
1425
1426                 port = &ports[pi];
1427                 if (rte_atomic16_cmpset(&(port->port_status),
1428                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
1429                         printf("Port %d is now not stopped\n", pi);
1430                         continue;
1431                 }
1432
1433                 rte_eth_dev_close(pi);
1434
1435                 if (rte_atomic16_cmpset(&(port->port_status),
1436                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
1437                         printf("Port %d can not be set into stopped\n", pi);
1438         }
1439
1440         printf("Done\n");
1441 }
1442
1443 int
1444 all_ports_stopped(void)
1445 {
1446         portid_t pi;
1447         struct rte_port *port;
1448
1449         for (pi = 0; pi < nb_ports; pi++) {
1450                 port = &ports[pi];
1451                 if (port->port_status != RTE_PORT_STOPPED)
1452                         return 0;
1453         }
1454
1455         return 1;
1456 }
1457
1458 void
1459 pmd_test_exit(void)
1460 {
1461         portid_t pt_id;
1462
1463         for (pt_id = 0; pt_id < nb_ports; pt_id++) {
1464                 printf("Stopping port %d...", pt_id);
1465                 fflush(stdout);
1466                 rte_eth_dev_close(pt_id);
1467                 printf("done\n");
1468         }
1469         printf("bye...\n");
1470 }
1471
1472 typedef void (*cmd_func_t)(void);
1473 struct pmd_test_command {
1474         const char *cmd_name;
1475         cmd_func_t cmd_func;
1476 };
1477
1478 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
1479
1480 /* Check the link status of all ports in up to 9s, and print them finally */
1481 static void
1482 check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
1483 {
1484 #define CHECK_INTERVAL 100 /* 100ms */
1485 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
1486         uint8_t portid, count, all_ports_up, print_flag = 0;
1487         struct rte_eth_link link;
1488
1489         printf("Checking link statuses...\n");
1490         fflush(stdout);
1491         for (count = 0; count <= MAX_CHECK_TIME; count++) {
1492                 all_ports_up = 1;
1493                 for (portid = 0; portid < port_num; portid++) {
1494                         if ((port_mask & (1 << portid)) == 0)
1495                                 continue;
1496                         memset(&link, 0, sizeof(link));
1497                         rte_eth_link_get_nowait(portid, &link);
1498                         /* print link status if flag set */
1499                         if (print_flag == 1) {
1500                                 if (link.link_status)
1501                                         printf("Port %d Link Up - speed %u "
1502                                                 "Mbps - %s\n", (uint8_t)portid,
1503                                                 (unsigned)link.link_speed,
1504                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
1505                                         ("full-duplex") : ("half-duplex\n"));
1506                                 else
1507                                         printf("Port %d Link Down\n",
1508                                                 (uint8_t)portid);
1509                                 continue;
1510                         }
1511                         /* clear all_ports_up flag if any link down */
1512                         if (link.link_status == 0) {
1513                                 all_ports_up = 0;
1514                                 break;
1515                         }
1516                 }
1517                 /* after finally printing all link status, get out */
1518                 if (print_flag == 1)
1519                         break;
1520
1521                 if (all_ports_up == 0) {
1522                         fflush(stdout);
1523                         rte_delay_ms(CHECK_INTERVAL);
1524                 }
1525
1526                 /* set the print_flag if all ports up or timeout */
1527                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
1528                         print_flag = 1;
1529                 }
1530         }
1531 }
1532
1533 static int
1534 set_tx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port)
1535 {
1536         uint16_t i;
1537         int diag;
1538         uint8_t mapping_found = 0;
1539
1540         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
1541                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
1542                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
1543                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
1544                                         tx_queue_stats_mappings[i].queue_id,
1545                                         tx_queue_stats_mappings[i].stats_counter_id);
1546                         if (diag != 0)
1547                                 return diag;
1548                         mapping_found = 1;
1549                 }
1550         }
1551         if (mapping_found)
1552                 port->tx_queue_stats_mapping_enabled = 1;
1553         return 0;
1554 }
1555
1556 static int
1557 set_rx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port)
1558 {
1559         uint16_t i;
1560         int diag;
1561         uint8_t mapping_found = 0;
1562
1563         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
1564                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
1565                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
1566                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
1567                                         rx_queue_stats_mappings[i].queue_id,
1568                                         rx_queue_stats_mappings[i].stats_counter_id);
1569                         if (diag != 0)
1570                                 return diag;
1571                         mapping_found = 1;
1572                 }
1573         }
1574         if (mapping_found)
1575                 port->rx_queue_stats_mapping_enabled = 1;
1576         return 0;
1577 }
1578
1579 static void
1580 map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port)
1581 {
1582         int diag = 0;
1583
1584         diag = set_tx_queue_stats_mapping_registers(pi, port);
1585         if (diag != 0) {
1586                 if (diag == -ENOTSUP) {
1587                         port->tx_queue_stats_mapping_enabled = 0;
1588                         printf("TX queue stats mapping not supported port id=%d\n", pi);
1589                 }
1590                 else
1591                         rte_exit(EXIT_FAILURE,
1592                                         "set_tx_queue_stats_mapping_registers "
1593                                         "failed for port id=%d diag=%d\n",
1594                                         pi, diag);
1595         }
1596
1597         diag = set_rx_queue_stats_mapping_registers(pi, port);
1598         if (diag != 0) {
1599                 if (diag == -ENOTSUP) {
1600                         port->rx_queue_stats_mapping_enabled = 0;
1601                         printf("RX queue stats mapping not supported port id=%d\n", pi);
1602                 }
1603                 else
1604                         rte_exit(EXIT_FAILURE,
1605                                         "set_rx_queue_stats_mapping_registers "
1606                                         "failed for port id=%d diag=%d\n",
1607                                         pi, diag);
1608         }
1609 }
1610
1611 void
1612 init_port_config(void)
1613 {
1614         portid_t pid;
1615         struct rte_port *port;
1616
1617         for (pid = 0; pid < nb_ports; pid++) {
1618                 port = &ports[pid];
1619                 port->dev_conf.rxmode = rx_mode;
1620                 port->dev_conf.fdir_conf = fdir_conf;
1621                 if (nb_rxq > 1) {
1622                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1623                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = rss_hf;
1624                 } else {
1625                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1626                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
1627                 }
1628
1629                 /* In SR-IOV mode, RSS mode is not available */
1630                 if (port->dcb_flag == 0 && port->dev_info.max_vfs == 0) {
1631                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
1632                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
1633                         else
1634                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
1635                 }
1636
1637                 port->rx_conf.rx_thresh = rx_thresh;
1638                 port->rx_conf.rx_free_thresh = rx_free_thresh;
1639                 port->rx_conf.rx_drop_en = rx_drop_en;
1640                 port->tx_conf.tx_thresh = tx_thresh;
1641                 port->tx_conf.tx_rs_thresh = tx_rs_thresh;
1642                 port->tx_conf.tx_free_thresh = tx_free_thresh;
1643                 port->tx_conf.txq_flags = txq_flags;
1644
1645                 rte_eth_macaddr_get(pid, &port->eth_addr);
1646
1647                 map_port_queue_stats_mapping_registers(pid, port);
1648 #ifdef RTE_NIC_BYPASS
1649                 rte_eth_dev_bypass_init(pid);
1650 #endif
1651         }
1652 }
1653
1654 const uint16_t vlan_tags[] = {
1655                 0,  1,  2,  3,  4,  5,  6,  7,
1656                 8,  9, 10, 11,  12, 13, 14, 15,
1657                 16, 17, 18, 19, 20, 21, 22, 23,
1658                 24, 25, 26, 27, 28, 29, 30, 31
1659 };
1660
1661 static  int
1662 get_eth_dcb_conf(struct rte_eth_conf *eth_conf, struct dcb_config *dcb_conf)
1663 {
1664         uint8_t i;
1665
1666         /*
1667          * Builds up the correct configuration for dcb+vt based on the vlan tags array
1668          * given above, and the number of traffic classes available for use.
1669          */
1670         if (dcb_conf->dcb_mode == DCB_VT_ENABLED) {
1671                 struct rte_eth_vmdq_dcb_conf vmdq_rx_conf;
1672                 struct rte_eth_vmdq_dcb_tx_conf vmdq_tx_conf;
1673
1674                 /* VMDQ+DCB RX and TX configrations */
1675                 vmdq_rx_conf.enable_default_pool = 0;
1676                 vmdq_rx_conf.default_pool = 0;
1677                 vmdq_rx_conf.nb_queue_pools =
1678                         (dcb_conf->num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
1679                 vmdq_tx_conf.nb_queue_pools =
1680                         (dcb_conf->num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
1681
1682                 vmdq_rx_conf.nb_pool_maps = sizeof( vlan_tags )/sizeof( vlan_tags[ 0 ]);
1683                 for (i = 0; i < vmdq_rx_conf.nb_pool_maps; i++) {
1684                         vmdq_rx_conf.pool_map[i].vlan_id = vlan_tags[ i ];
1685                         vmdq_rx_conf.pool_map[i].pools = 1 << (i % vmdq_rx_conf.nb_queue_pools);
1686                 }
1687                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
1688                         vmdq_rx_conf.dcb_queue[i] = i;
1689                         vmdq_tx_conf.dcb_queue[i] = i;
1690                 }
1691
1692                 /*set DCB mode of RX and TX of multiple queues*/
1693                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
1694                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
1695                 if (dcb_conf->pfc_en)
1696                         eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT|ETH_DCB_PFC_SUPPORT;
1697                 else
1698                         eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
1699
1700                 (void)(rte_memcpy(&eth_conf->rx_adv_conf.vmdq_dcb_conf, &vmdq_rx_conf,
1701                                 sizeof(struct rte_eth_vmdq_dcb_conf)));
1702                 (void)(rte_memcpy(&eth_conf->tx_adv_conf.vmdq_dcb_tx_conf, &vmdq_tx_conf,
1703                                 sizeof(struct rte_eth_vmdq_dcb_tx_conf)));
1704         }
1705         else {
1706                 struct rte_eth_dcb_rx_conf rx_conf;
1707                 struct rte_eth_dcb_tx_conf tx_conf;
1708
1709                 /* queue mapping configuration of DCB RX and TX */
1710                 if (dcb_conf->num_tcs == ETH_4_TCS)
1711                         dcb_q_mapping = DCB_4_TCS_Q_MAPPING;
1712                 else
1713                         dcb_q_mapping = DCB_8_TCS_Q_MAPPING;
1714
1715                 rx_conf.nb_tcs = dcb_conf->num_tcs;
1716                 tx_conf.nb_tcs = dcb_conf->num_tcs;
1717
1718                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++){
1719                         rx_conf.dcb_queue[i] = i;
1720                         tx_conf.dcb_queue[i] = i;
1721                 }
1722                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB;
1723                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
1724                 if (dcb_conf->pfc_en)
1725                         eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT|ETH_DCB_PFC_SUPPORT;
1726                 else
1727                         eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
1728
1729                 (void)(rte_memcpy(&eth_conf->rx_adv_conf.dcb_rx_conf, &rx_conf,
1730                                 sizeof(struct rte_eth_dcb_rx_conf)));
1731                 (void)(rte_memcpy(&eth_conf->tx_adv_conf.dcb_tx_conf, &tx_conf,
1732                                 sizeof(struct rte_eth_dcb_tx_conf)));
1733         }
1734
1735         return 0;
1736 }
1737
1738 int
1739 init_port_dcb_config(portid_t pid,struct dcb_config *dcb_conf)
1740 {
1741         struct rte_eth_conf port_conf;
1742         struct rte_port *rte_port;
1743         int retval;
1744         uint16_t nb_vlan;
1745         uint16_t i;
1746
1747         /* rxq and txq configuration in dcb mode */
1748         nb_rxq = 128;
1749         nb_txq = 128;
1750         rx_free_thresh = 64;
1751
1752         memset(&port_conf,0,sizeof(struct rte_eth_conf));
1753         /* Enter DCB configuration status */
1754         dcb_config = 1;
1755
1756         nb_vlan = sizeof( vlan_tags )/sizeof( vlan_tags[ 0 ]);
1757         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
1758         retval = get_eth_dcb_conf(&port_conf, dcb_conf);
1759         if (retval < 0)
1760                 return retval;
1761
1762         rte_port = &ports[pid];
1763         memcpy(&rte_port->dev_conf, &port_conf,sizeof(struct rte_eth_conf));
1764
1765         rte_port->rx_conf.rx_thresh = rx_thresh;
1766         rte_port->rx_conf.rx_free_thresh = rx_free_thresh;
1767         rte_port->tx_conf.tx_thresh = tx_thresh;
1768         rte_port->tx_conf.tx_rs_thresh = tx_rs_thresh;
1769         rte_port->tx_conf.tx_free_thresh = tx_free_thresh;
1770         /* VLAN filter */
1771         rte_port->dev_conf.rxmode.hw_vlan_filter = 1;
1772         for (i = 0; i < nb_vlan; i++){
1773                 rx_vft_set(pid, vlan_tags[i], 1);
1774         }
1775
1776         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
1777         map_port_queue_stats_mapping_registers(pid, rte_port);
1778
1779         rte_port->dcb_flag = 1;
1780
1781         return 0;
1782 }
1783
1784 #ifdef RTE_EXEC_ENV_BAREMETAL
1785 #define main _main
1786 #endif
1787
1788 int
1789 main(int argc, char** argv)
1790 {
1791         int  diag;
1792         uint8_t port_id;
1793
1794         diag = rte_eal_init(argc, argv);
1795         if (diag < 0)
1796                 rte_panic("Cannot init EAL\n");
1797
1798         if (rte_pmd_init_all())
1799                 rte_panic("Cannot init PMD\n");
1800
1801         if (rte_eal_pci_probe())
1802                 rte_panic("Cannot probe PCI\n");
1803
1804         nb_ports = (portid_t) rte_eth_dev_count();
1805         if (nb_ports == 0)
1806                 rte_exit(EXIT_FAILURE, "No probed ethernet devices - "
1807                                                         "check that "
1808                           "CONFIG_RTE_LIBRTE_IGB_PMD=y and that "
1809                           "CONFIG_RTE_LIBRTE_EM_PMD=y and that "
1810                           "CONFIG_RTE_LIBRTE_IXGBE_PMD=y in your "
1811                           "configuration file\n");
1812
1813         set_def_fwd_config();
1814         if (nb_lcores == 0)
1815                 rte_panic("Empty set of forwarding logical cores - check the "
1816                           "core mask supplied in the command parameters\n");
1817
1818         argc -= diag;
1819         argv += diag;
1820         if (argc > 1)
1821                 launch_args_parse(argc, argv);
1822
1823         if (nb_rxq > nb_txq)
1824                 printf("Warning: nb_rxq=%d enables RSS configuration, "
1825                        "but nb_txq=%d will prevent to fully test it.\n",
1826                        nb_rxq, nb_txq);
1827
1828         init_config();
1829         if (start_port(RTE_PORT_ALL) != 0)
1830                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
1831
1832         /* set all ports to promiscuous mode by default */
1833         for (port_id = 0; port_id < nb_ports; port_id++)
1834                 rte_eth_promiscuous_enable(port_id);
1835
1836 #ifdef RTE_LIBRTE_CMDLINE
1837         if (interactive == 1) {
1838                 if (auto_start) {
1839                         printf("Start automatic packet forwarding\n");
1840                         start_packet_forwarding(0);
1841                 }
1842                 prompt();
1843         } else
1844 #endif
1845         {
1846                 char c;
1847                 int rc;
1848
1849                 printf("No commandline core given, start packet forwarding\n");
1850                 start_packet_forwarding(0);
1851                 printf("Press enter to exit\n");
1852                 rc = read(0, &c, 1);
1853                 if (rc < 0)
1854                         return 1;
1855         }
1856
1857         return 0;
1858 }