app/test-pmd/testpmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdarg.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <signal.h>
  38 #include <string.h>
  39 #include <time.h>
  40 #include <fcntl.h>
  41 #include <sys/types.h>
  42 #include <errno.h>
  43
  44 #include <sys/queue.h>
  45 #include <sys/stat.h>
  46
  47 #include <stdint.h>
  48 #include <unistd.h>
  49 #include <inttypes.h>
  50
  51 #include <rte_common.h>
  52 #include <rte_errno.h>
  53 #include <rte_byteorder.h>
  54 #include <rte_log.h>
  55 #include <rte_debug.h>
  56 #include <rte_cycles.h>
  57 #include <rte_memory.h>
  58 #include <rte_memcpy.h>
  59 #include <rte_memzone.h>
  60 #include <rte_launch.h>
  61 #include <rte_eal.h>
  62 #include <rte_per_lcore.h>
  63 #include <rte_lcore.h>
  64 #include <rte_atomic.h>
  65 #include <rte_branch_prediction.h>
  66 #include <rte_ring.h>
  67 #include <rte_mempool.h>
  68 #include <rte_malloc.h>
  69 #include <rte_mbuf.h>
  70 #include <rte_interrupts.h>
  71 #include <rte_pci.h>
  72 #include <rte_ether.h>
  73 #include <rte_ethdev.h>
  74 #include <rte_dev.h>
  75 #include <rte_string_fns.h>
  76 #ifdef RTE_LIBRTE_PMD_XENVIRT
  77 #include <rte_eth_xenvirt.h>
  78 #endif
  79
  80 #include "testpmd.h"
  81
  82 uint16_t verbose_level = 0; /**< Silent by default. */
  83
  84 /* use master core for command line ? */
  85 uint8_t interactive = 0;
  86 uint8_t auto_start = 0;
  87
  88 /*
  89  * NUMA support configuration.
  90  * When set, the NUMA support attempts to dispatch the allocation of the
  91  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  92  * probed ports among the CPU sockets 0 and 1.
  93  * Otherwise, all memory is allocated from CPU socket 0.
  94  */
  95 uint8_t numa_support = 0; /**< No numa support by default */
  96
  97 /*
  98  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
  99  * not configured.
 100  */
 101 uint8_t socket_num = UMA_NO_CONFIG;
 102
 103 /*
 104  * Use ANONYMOUS mapped memory (might be not physically continuous) for mbufs.
 105  */
 106 uint8_t mp_anon = 0;
 107
 108 /*
 109  * Record the Ethernet address of peer target ports to which packets are
 110  * forwarded.
 111  * Must be instanciated with the ethernet addresses of peer traffic generator
 112  * ports.
 113  */
 114 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 115 portid_t nb_peer_eth_addrs = 0;
 116
 117 /*
 118  * Probed Target Environment.
 119  */
 120 struct rte_port *ports;        /**< For all probed ethernet ports. */
 121 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 122 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 123 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 124
 125 /*
 126  * Test Forwarding Configuration.
 127  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 128  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 129  */
 130 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 131 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 132 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 133 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 134
 135 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 136 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 137
 138 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 139 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 140
 141 /*
 142  * Forwarding engines.
 143  */
 144 struct fwd_engine * fwd_engines[] = {
 145         &io_fwd_engine,
 146         &mac_fwd_engine,
 147         &mac_swap_engine,
 148         &flow_gen_engine,
 149         &rx_only_engine,
 150         &tx_only_engine,
 151         &csum_fwd_engine,
 152         &icmp_echo_engine,
 153 #ifdef RTE_LIBRTE_IEEE1588
 154         &ieee1588_fwd_engine,
 155 #endif
 156         NULL,
 157 };
 158
 159 struct fwd_config cur_fwd_config;
 160 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 161 uint32_t retry_enabled;
 162 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 163 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 164
 165 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 166 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 167                                       * specified on command-line. */
 168
 169 /*
 170  * Configuration of packet segments used by the "txonly" processing engine.
 171  */
 172 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 173 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 174         TXONLY_DEF_PACKET_LEN,
 175 };
 176 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 177
 178 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 179 /**< Split policy for packets to TX. */
 180
 181 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 182 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 183
 184 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 185 uint8_t dcb_config = 0;
 186
 187 /* Whether the dcb is in testing status */
 188 uint8_t dcb_test = 0;
 189
 190 /*
 191  * Configurable number of RX/TX queues.
 192  */
 193 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 194 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 195
 196 /*
 197  * Configurable number of RX/TX ring descriptors.
 198  */
 199 #define RTE_TEST_RX_DESC_DEFAULT 128
 200 #define RTE_TEST_TX_DESC_DEFAULT 512
 201 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 202 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 203
 204 #define RTE_PMD_PARAM_UNSET -1
 205 /*
 206  * Configurable values of RX and TX ring threshold registers.
 207  */
 208
 209 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 210 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 211 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 212
 213 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 214 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 215 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 216
 217 /*
 218  * Configurable value of RX free threshold.
 219  */
 220 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 221
 222 /*
 223  * Configurable value of RX drop enable.
 224  */
 225 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 226
 227 /*
 228  * Configurable value of TX free threshold.
 229  */
 230 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 231
 232 /*
 233  * Configurable value of TX RS bit threshold.
 234  */
 235 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 236
 237 /*
 238  * Configurable value of TX queue flags.
 239  */
 240 int32_t txq_flags = RTE_PMD_PARAM_UNSET;
 241
 242 /*
 243  * Receive Side Scaling (RSS) configuration.
 244  */
 245 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 246
 247 /*
 248  * Port topology configuration
 249  */
 250 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 251
 252 /*
 253  * Avoids to flush all the RX streams before starts forwarding.
 254  */
 255 uint8_t no_flush_rx = 0; /* flush by default */
 256
 257 /*
 258  * Avoids to check link status when starting/stopping a port.
 259  */
 260 uint8_t no_link_check = 0; /* check by default */
 261
 262 /*
 263  * NIC bypass mode configuration options.
 264  */
 265 #ifdef RTE_NIC_BYPASS
 266
 267 /* The NIC bypass watchdog timeout. */
 268 uint32_t bypass_timeout = RTE_BYPASS_TMT_OFF;
 269
 270 #endif
 271
 272 /*
 273  * Ethernet device configuration.
 274  */
 275 struct rte_eth_rxmode rx_mode = {
 276         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 277         .split_hdr_size = 0,
 278         .header_split   = 0, /**< Header Split disabled. */
 279         .hw_ip_checksum = 0, /**< IP checksum offload disabled. */
 280         .hw_vlan_filter = 1, /**< VLAN filtering enabled. */
 281         .hw_vlan_strip  = 1, /**< VLAN strip enabled. */
 282         .hw_vlan_extend = 0, /**< Extended VLAN disabled. */
 283         .jumbo_frame    = 0, /**< Jumbo Frame Support disabled. */
 284         .hw_strip_crc   = 0, /**< CRC stripping by hardware disabled. */
 285 };
 286
 287 struct rte_fdir_conf fdir_conf = {
 288         .mode = RTE_FDIR_MODE_NONE,
 289         .pballoc = RTE_FDIR_PBALLOC_64K,
 290         .status = RTE_FDIR_REPORT_STATUS,
 291         .mask = {
 292                 .vlan_tci_mask = 0x0,
 293                 .ipv4_mask     = {
 294                         .src_ip = 0xFFFFFFFF,
 295                         .dst_ip = 0xFFFFFFFF,
 296                 },
 297                 .ipv6_mask     = {
 298                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 299                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 300                 },
 301                 .src_port_mask = 0xFFFF,
 302                 .dst_port_mask = 0xFFFF,
 303                 .mac_addr_byte_mask = 0xFF,
 304                 .tunnel_type_mask = 1,
 305                 .tunnel_id_mask = 0xFFFFFFFF,
 306         },
 307         .drop_queue = 127,
 308 };
 309
 310 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 311
 312 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 313 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 314
 315 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 316 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 317
 318 uint16_t nb_tx_queue_stats_mappings = 0;
 319 uint16_t nb_rx_queue_stats_mappings = 0;
 320
 321 unsigned max_socket = 0;
 322
 323 /* Forward function declarations */
 324 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port);
 325 static void check_all_ports_link_status(uint32_t port_mask);
 326
 327 /*
 328  * Check if all the ports are started.
 329  * If yes, return positive value. If not, return zero.
 330  */
 331 static int all_ports_started(void);
 332
 333 /*
 334  * Find next enabled port
 335  */
 336 portid_t
 337 find_next_port(portid_t p, struct rte_port *ports, int size)
 338 {
 339         if (ports == NULL)
 340                 rte_exit(-EINVAL, "failed to find a next port id\n");
 341
 342         while ((p < size) && (ports[p].enabled == 0))
 343                 p++;
 344         return p;
 345 }
 346
 347 /*
 348  * Setup default configuration.
 349  */
 350 static void
 351 set_default_fwd_lcores_config(void)
 352 {
 353         unsigned int i;
 354         unsigned int nb_lc;
 355         unsigned int sock_num;
 356
 357         nb_lc = 0;
 358         for (i = 0; i < RTE_MAX_LCORE; i++) {
 359                 sock_num = rte_lcore_to_socket_id(i) + 1;
 360                 if (sock_num > max_socket) {
 361                         if (sock_num > RTE_MAX_NUMA_NODES)
 362                                 rte_exit(EXIT_FAILURE, "Total sockets greater than %u\n", RTE_MAX_NUMA_NODES);
 363                         max_socket = sock_num;
 364                 }
 365                 if (!rte_lcore_is_enabled(i))
 366                         continue;
 367                 if (i == rte_get_master_lcore())
 368                         continue;
 369                 fwd_lcores_cpuids[nb_lc++] = i;
 370         }
 371         nb_lcores = (lcoreid_t) nb_lc;
 372         nb_cfg_lcores = nb_lcores;
 373         nb_fwd_lcores = 1;
 374 }
 375
 376 static void
 377 set_def_peer_eth_addrs(void)
 378 {
 379         portid_t i;
 380
 381         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 382                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 383                 peer_eth_addrs[i].addr_bytes[5] = i;
 384         }
 385 }
 386
 387 static void
 388 set_default_fwd_ports_config(void)
 389 {
 390         portid_t pt_id;
 391
 392         for (pt_id = 0; pt_id < nb_ports; pt_id++)
 393                 fwd_ports_ids[pt_id] = pt_id;
 394
 395         nb_cfg_ports = nb_ports;
 396         nb_fwd_ports = nb_ports;
 397 }
 398
 399 void
 400 set_def_fwd_config(void)
 401 {
 402         set_default_fwd_lcores_config();
 403         set_def_peer_eth_addrs();
 404         set_default_fwd_ports_config();
 405 }
 406
 407 /*
 408  * Configuration initialisation done once at init time.
 409  */
 410 static void
 411 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 412                  unsigned int socket_id)
 413 {
 414         char pool_name[RTE_MEMPOOL_NAMESIZE];
 415         struct rte_mempool *rte_mp = NULL;
 416         uint32_t mb_size;
 417
 418         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 419         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 420
 421         RTE_LOG(INFO, USER1,
 422                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 423                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 424
 425 #ifdef RTE_LIBRTE_PMD_XENVIRT
 426         rte_mp = rte_mempool_gntalloc_create(pool_name, nb_mbuf, mb_size,
 427                 (unsigned) mb_mempool_cache,
 428                 sizeof(struct rte_pktmbuf_pool_private),
 429                 rte_pktmbuf_pool_init, NULL,
 430                 rte_pktmbuf_init, NULL,
 431                 socket_id, 0);
 432 #endif
 433
 434         /* if the former XEN allocation failed fall back to normal allocation */
 435         if (rte_mp == NULL) {
 436                 if (mp_anon != 0) {
 437                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 438                                 mb_size, (unsigned) mb_mempool_cache,
 439                                 sizeof(struct rte_pktmbuf_pool_private),
 440                                 socket_id, 0);
 441
 442                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 443                                 rte_mempool_free(rte_mp);
 444                                 rte_mp = NULL;
 445                         }
 446                         rte_pktmbuf_pool_init(rte_mp, NULL);
 447                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 448                 } else {
 449                         /* wrapper to rte_mempool_create() */
 450                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 451                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 452                 }
 453         }
 454
 455         if (rte_mp == NULL) {
 456                 rte_exit(EXIT_FAILURE,
 457                         "Creation of mbuf pool for socket %u failed: %s\n",
 458                         socket_id, rte_strerror(rte_errno));
 459         } else if (verbose_level > 0) {
 460                 rte_mempool_dump(stdout, rte_mp);
 461         }
 462 }
 463
 464 /*
 465  * Check given socket id is valid or not with NUMA mode,
 466  * if valid, return 0, else return -1
 467  */
 468 static int
 469 check_socket_id(const unsigned int socket_id)
 470 {
 471         static int warning_once = 0;
 472
 473         if (socket_id >= max_socket) {
 474                 if (!warning_once && numa_support)
 475                         printf("Warning: NUMA should be configured manually by"
 476                                " using --port-numa-config and"
 477                                " --ring-numa-config parameters along with"
 478                                " --numa.\n");
 479                 warning_once = 1;
 480                 return -1;
 481         }
 482         return 0;
 483 }
 484
 485 static void
 486 init_config(void)
 487 {
 488         portid_t pid;
 489         struct rte_port *port;
 490         struct rte_mempool *mbp;
 491         unsigned int nb_mbuf_per_pool;
 492         lcoreid_t  lc_id;
 493         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
 494
 495         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
 496         /* Configuration of logical cores. */
 497         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
 498                                 sizeof(struct fwd_lcore *) * nb_lcores,
 499                                 RTE_CACHE_LINE_SIZE);
 500         if (fwd_lcores == NULL) {
 501                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
 502                                                         "failed\n", nb_lcores);
 503         }
 504         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 505                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
 506                                                sizeof(struct fwd_lcore),
 507                                                RTE_CACHE_LINE_SIZE);
 508                 if (fwd_lcores[lc_id] == NULL) {
 509                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
 510                                                                 "failed\n");
 511                 }
 512                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
 513         }
 514
 515         /*
 516          * Create pools of mbuf.
 517          * If NUMA support is disabled, create a single pool of mbuf in
 518          * socket 0 memory by default.
 519          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
 520          *
 521          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
 522          * nb_txd can be configured at run time.
 523          */
 524         if (param_total_num_mbufs)
 525                 nb_mbuf_per_pool = param_total_num_mbufs;
 526         else {
 527                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + (nb_lcores * mb_mempool_cache)
 528                                 + RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
 529
 530                 if (!numa_support)
 531                         nb_mbuf_per_pool =
 532                                 (nb_mbuf_per_pool * RTE_MAX_ETHPORTS);
 533         }
 534
 535         if (!numa_support) {
 536                 if (socket_num == UMA_NO_CONFIG)
 537                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0);
 538                 else
 539                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
 540                                                  socket_num);
 541         }
 542
 543         FOREACH_PORT(pid, ports) {
 544                 port = &ports[pid];
 545                 rte_eth_dev_info_get(pid, &port->dev_info);
 546
 547                 if (numa_support) {
 548                         if (port_numa[pid] != NUMA_NO_CONFIG)
 549                                 port_per_socket[port_numa[pid]]++;
 550                         else {
 551                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
 552
 553                                 /* if socket_id is invalid, set to 0 */
 554                                 if (check_socket_id(socket_id) < 0)
 555                                         socket_id = 0;
 556                                 port_per_socket[socket_id]++;
 557                         }
 558                 }
 559
 560                 /* set flag to initialize port/queue */
 561                 port->need_reconfig = 1;
 562                 port->need_reconfig_queues = 1;
 563         }
 564
 565         if (numa_support) {
 566                 uint8_t i;
 567                 unsigned int nb_mbuf;
 568
 569                 if (param_total_num_mbufs)
 570                         nb_mbuf_per_pool = nb_mbuf_per_pool/nb_ports;
 571
 572                 for (i = 0; i < max_socket; i++) {
 573                         nb_mbuf = (nb_mbuf_per_pool * RTE_MAX_ETHPORTS);
 574                         if (nb_mbuf)
 575                                 mbuf_pool_create(mbuf_data_size,
 576                                                 nb_mbuf,i);
 577                 }
 578         }
 579         init_port_config();
 580
 581         /*
 582          * Records which Mbuf pool to use by each logical core, if needed.
 583          */
 584         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 585                 mbp = mbuf_pool_find(
 586                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
 587
 588                 if (mbp == NULL)
 589                         mbp = mbuf_pool_find(0);
 590                 fwd_lcores[lc_id]->mbp = mbp;
 591         }
 592
 593         /* Configuration of packet forwarding streams. */
 594         if (init_fwd_streams() < 0)
 595                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 596
 597         fwd_config_setup();
 598 }
 599
 600
 601 void
 602 reconfig(portid_t new_port_id, unsigned socket_id)
 603 {
 604         struct rte_port *port;
 605
 606         /* Reconfiguration of Ethernet ports. */
 607         port = &ports[new_port_id];
 608         rte_eth_dev_info_get(new_port_id, &port->dev_info);
 609
 610         /* set flag to initialize port/queue */
 611         port->need_reconfig = 1;
 612         port->need_reconfig_queues = 1;
 613         port->socket_id = socket_id;
 614
 615         init_port_config();
 616 }
 617
 618
 619 int
 620 init_fwd_streams(void)
 621 {
 622         portid_t pid;
 623         struct rte_port *port;
 624         streamid_t sm_id, nb_fwd_streams_new;
 625         queueid_t q;
 626
 627         /* set socket id according to numa or not */
 628         FOREACH_PORT(pid, ports) {
 629                 port = &ports[pid];
 630                 if (nb_rxq > port->dev_info.max_rx_queues) {
 631                         printf("Fail: nb_rxq(%d) is greater than "
 632                                 "max_rx_queues(%d)\n", nb_rxq,
 633                                 port->dev_info.max_rx_queues);
 634                         return -1;
 635                 }
 636                 if (nb_txq > port->dev_info.max_tx_queues) {
 637                         printf("Fail: nb_txq(%d) is greater than "
 638                                 "max_tx_queues(%d)\n", nb_txq,
 639                                 port->dev_info.max_tx_queues);
 640                         return -1;
 641                 }
 642                 if (numa_support) {
 643                         if (port_numa[pid] != NUMA_NO_CONFIG)
 644                                 port->socket_id = port_numa[pid];
 645                         else {
 646                                 port->socket_id = rte_eth_dev_socket_id(pid);
 647
 648                                 /* if socket_id is invalid, set to 0 */
 649                                 if (check_socket_id(port->socket_id) < 0)
 650                                         port->socket_id = 0;
 651                         }
 652                 }
 653                 else {
 654                         if (socket_num == UMA_NO_CONFIG)
 655                                 port->socket_id = 0;
 656                         else
 657                                 port->socket_id = socket_num;
 658                 }
 659         }
 660
 661         q = RTE_MAX(nb_rxq, nb_txq);
 662         if (q == 0) {
 663                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
 664                 return -1;
 665         }
 666         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
 667         if (nb_fwd_streams_new == nb_fwd_streams)
 668                 return 0;
 669         /* clear the old */
 670         if (fwd_streams != NULL) {
 671                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 672                         if (fwd_streams[sm_id] == NULL)
 673                                 continue;
 674                         rte_free(fwd_streams[sm_id]);
 675                         fwd_streams[sm_id] = NULL;
 676                 }
 677                 rte_free(fwd_streams);
 678                 fwd_streams = NULL;
 679         }
 680
 681         /* init new */
 682         nb_fwd_streams = nb_fwd_streams_new;
 683         fwd_streams = rte_zmalloc("testpmd: fwd_streams",
 684                 sizeof(struct fwd_stream *) * nb_fwd_streams, RTE_CACHE_LINE_SIZE);
 685         if (fwd_streams == NULL)
 686                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_stream *)) "
 687                                                 "failed\n", nb_fwd_streams);
 688
 689         for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 690                 fwd_streams[sm_id] = rte_zmalloc("testpmd: struct fwd_stream",
 691                                 sizeof(struct fwd_stream), RTE_CACHE_LINE_SIZE);
 692                 if (fwd_streams[sm_id] == NULL)
 693                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_stream)"
 694                                                                 " failed\n");
 695         }
 696
 697         return 0;
 698 }
 699
 700 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 701 static void
 702 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
 703 {
 704         unsigned int total_burst;
 705         unsigned int nb_burst;
 706         unsigned int burst_stats[3];
 707         uint16_t pktnb_stats[3];
 708         uint16_t nb_pkt;
 709         int burst_percent[3];
 710
 711         /*
 712          * First compute the total number of packet bursts and the
 713          * two highest numbers of bursts of the same number of packets.
 714          */
 715         total_burst = 0;
 716         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
 717         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
 718         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
 719                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
 720                 if (nb_burst == 0)
 721                         continue;
 722                 total_burst += nb_burst;
 723                 if (nb_burst > burst_stats[0]) {
 724                         burst_stats[1] = burst_stats[0];
 725                         pktnb_stats[1] = pktnb_stats[0];
 726                         burst_stats[0] = nb_burst;
 727                         pktnb_stats[0] = nb_pkt;
 728                 }
 729         }
 730         if (total_burst == 0)
 731                 return;
 732         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
 733         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
 734                burst_percent[0], (int) pktnb_stats[0]);
 735         if (burst_stats[0] == total_burst) {
 736                 printf("]\n");
 737                 return;
 738         }
 739         if (burst_stats[0] + burst_stats[1] == total_burst) {
 740                 printf(" + %d%% of %d pkts]\n",
 741                        100 - burst_percent[0], pktnb_stats[1]);
 742                 return;
 743         }
 744         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
 745         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
 746         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
 747                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
 748                 return;
 749         }
 750         printf(" + %d%% of %d pkts + %d%% of others]\n",
 751                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
 752 }
 753 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
 754
 755 static void
 756 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats)
 757 {
 758         struct rte_port *port;
 759         uint8_t i;
 760
 761         static const char *fwd_stats_border = "----------------------";
 762
 763         port = &ports[port_id];
 764         printf("\n  %s Forward statistics for port %-2d %s\n",
 765                fwd_stats_border, port_id, fwd_stats_border);
 766
 767         if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) {
 768                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
 769                        "%-"PRIu64"\n",
 770                        stats->ipackets, stats->imissed,
 771                        (uint64_t) (stats->ipackets + stats->imissed));
 772
 773                 if (cur_fwd_eng == &csum_fwd_engine)
 774                         printf("  Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n",
 775                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 776                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
 777                         printf("  RX-error: %-"PRIu64"\n",  stats->ierrors);
 778                         printf("  RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf);
 779                 }
 780
 781                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
 782                        "%-"PRIu64"\n",
 783                        stats->opackets, port->tx_dropped,
 784                        (uint64_t) (stats->opackets + port->tx_dropped));
 785         }
 786         else {
 787                 printf("  RX-packets:             %14"PRIu64"    RX-dropped:%14"PRIu64"    RX-total:"
 788                        "%14"PRIu64"\n",
 789                        stats->ipackets, stats->imissed,
 790                        (uint64_t) (stats->ipackets + stats->imissed));
 791
 792                 if (cur_fwd_eng == &csum_fwd_engine)
 793                         printf("  Bad-ipcsum:%14"PRIu64"    Bad-l4csum:%14"PRIu64"\n",
 794                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 795                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
 796                         printf("  RX-error:%"PRIu64"\n", stats->ierrors);
 797                         printf("  RX-nombufs:             %14"PRIu64"\n",
 798                                stats->rx_nombuf);
 799                 }
 800
 801                 printf("  TX-packets:             %14"PRIu64"    TX-dropped:%14"PRIu64"    TX-total:"
 802                        "%14"PRIu64"\n",
 803                        stats->opackets, port->tx_dropped,
 804                        (uint64_t) (stats->opackets + port->tx_dropped));
 805         }
 806
 807 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 808         if (port->rx_stream)
 809                 pkt_burst_stats_display("RX",
 810                         &port->rx_stream->rx_burst_stats);
 811         if (port->tx_stream)
 812                 pkt_burst_stats_display("TX",
 813                         &port->tx_stream->tx_burst_stats);
 814 #endif
 815
 816         if (port->rx_queue_stats_mapping_enabled) {
 817                 printf("\n");
 818                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 819                         printf("  Stats reg %2d RX-packets:%14"PRIu64
 820                                "     RX-errors:%14"PRIu64
 821                                "    RX-bytes:%14"PRIu64"\n",
 822                                i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]);
 823                 }
 824                 printf("\n");
 825         }
 826         if (port->tx_queue_stats_mapping_enabled) {
 827                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 828                         printf("  Stats reg %2d TX-packets:%14"PRIu64
 829                                "                                 TX-bytes:%14"PRIu64"\n",
 830                                i, stats->q_opackets[i], stats->q_obytes[i]);
 831                 }
 832         }
 833
 834         printf("  %s--------------------------------%s\n",
 835                fwd_stats_border, fwd_stats_border);
 836 }
 837
 838 static void
 839 fwd_stream_stats_display(streamid_t stream_id)
 840 {
 841         struct fwd_stream *fs;
 842         static const char *fwd_top_stats_border = "-------";
 843
 844         fs = fwd_streams[stream_id];
 845         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
 846             (fs->fwd_dropped == 0))
 847                 return;
 848         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
 849                "TX Port=%2d/Queue=%2d %s\n",
 850                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
 851                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
 852         printf("  RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u",
 853                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
 854
 855         /* if checksum mode */
 856         if (cur_fwd_eng == &csum_fwd_engine) {
 857                printf("  RX- bad IP checksum: %-14u  Rx- bad L4 checksum: "
 858                         "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum);
 859         }
 860
 861 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 862         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
 863         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
 864 #endif
 865 }
 866
 867 static void
 868 flush_fwd_rx_queues(void)
 869 {
 870         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 871         portid_t  rxp;
 872         portid_t port_id;
 873         queueid_t rxq;
 874         uint16_t  nb_rx;
 875         uint16_t  i;
 876         uint8_t   j;
 877
 878         for (j = 0; j < 2; j++) {
 879                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
 880                         for (rxq = 0; rxq < nb_rxq; rxq++) {
 881                                 port_id = fwd_ports_ids[rxp];
 882                                 do {
 883                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
 884                                                 pkts_burst, MAX_PKT_BURST);
 885                                         for (i = 0; i < nb_rx; i++)
 886                                                 rte_pktmbuf_free(pkts_burst[i]);
 887                                 } while (nb_rx > 0);
 888                         }
 889                 }
 890                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
 891         }
 892 }
 893
 894 static void
 895 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
 896 {
 897         struct fwd_stream **fsm;
 898         streamid_t nb_fs;
 899         streamid_t sm_id;
 900
 901         fsm = &fwd_streams[fc->stream_idx];
 902         nb_fs = fc->stream_nb;
 903         do {
 904                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
 905                         (*pkt_fwd)(fsm[sm_id]);
 906         } while (! fc->stopped);
 907 }
 908
 909 static int
 910 start_pkt_forward_on_core(void *fwd_arg)
 911 {
 912         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
 913                              cur_fwd_config.fwd_eng->packet_fwd);
 914         return 0;
 915 }
 916
 917 /*
 918  * Run the TXONLY packet forwarding engine to send a single burst of packets.
 919  * Used to start communication flows in network loopback test configurations.
 920  */
 921 static int
 922 run_one_txonly_burst_on_core(void *fwd_arg)
 923 {
 924         struct fwd_lcore *fwd_lc;
 925         struct fwd_lcore tmp_lcore;
 926
 927         fwd_lc = (struct fwd_lcore *) fwd_arg;
 928         tmp_lcore = *fwd_lc;
 929         tmp_lcore.stopped = 1;
 930         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
 931         return 0;
 932 }
 933
 934 /*
 935  * Launch packet forwarding:
 936  *     - Setup per-port forwarding context.
 937  *     - launch logical cores with their forwarding configuration.
 938  */
 939 static void
 940 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
 941 {
 942         port_fwd_begin_t port_fwd_begin;
 943         unsigned int i;
 944         unsigned int lc_id;
 945         int diag;
 946
 947         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
 948         if (port_fwd_begin != NULL) {
 949                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
 950                         (*port_fwd_begin)(fwd_ports_ids[i]);
 951         }
 952         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
 953                 lc_id = fwd_lcores_cpuids[i];
 954                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
 955                         fwd_lcores[i]->stopped = 0;
 956                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
 957                                                      fwd_lcores[i], lc_id);
 958                         if (diag != 0)
 959                                 printf("launch lcore %u failed - diag=%d\n",
 960                                        lc_id, diag);
 961                 }
 962         }
 963 }
 964
 965 /*
 966  * Launch packet forwarding configuration.
 967  */
 968 void
 969 start_packet_forwarding(int with_tx_first)
 970 {
 971         port_fwd_begin_t port_fwd_begin;
 972         port_fwd_end_t  port_fwd_end;
 973         struct rte_port *port;
 974         unsigned int i;
 975         portid_t   pt_id;
 976         streamid_t sm_id;
 977
 978         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
 979                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
 980
 981         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
 982                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
 983
 984         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
 985                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
 986                 (!nb_rxq || !nb_txq))
 987                 rte_exit(EXIT_FAILURE,
 988                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
 989                         cur_fwd_eng->fwd_mode_name);
 990
 991         if (all_ports_started() == 0) {
 992                 printf("Not all ports were started\n");
 993                 return;
 994         }
 995         if (test_done == 0) {
 996                 printf("Packet forwarding already started\n");
 997                 return;
 998         }
 999
1000         if (init_fwd_streams() < 0) {
1001                 printf("Fail from init_fwd_streams()\n");
1002                 return;
1003         }
1004
1005         if(dcb_test) {
1006                 for (i = 0; i < nb_fwd_ports; i++) {
1007                         pt_id = fwd_ports_ids[i];
1008                         port = &ports[pt_id];
1009                         if (!port->dcb_flag) {
1010                                 printf("In DCB mode, all forwarding ports must "
1011                                        "be configured in this mode.\n");
1012                                 return;
1013                         }
1014                 }
1015                 if (nb_fwd_lcores == 1) {
1016                         printf("In DCB mode,the nb forwarding cores "
1017                                "should be larger than 1.\n");
1018                         return;
1019                 }
1020         }
1021         test_done = 0;
1022
1023         if(!no_flush_rx)
1024                 flush_fwd_rx_queues();
1025
1026         fwd_config_setup();
1027         rxtx_config_display();
1028
1029         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1030                 pt_id = fwd_ports_ids[i];
1031                 port = &ports[pt_id];
1032                 rte_eth_stats_get(pt_id, &port->stats);
1033                 port->tx_dropped = 0;
1034
1035                 map_port_queue_stats_mapping_registers(pt_id, port);
1036         }
1037         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1038                 fwd_streams[sm_id]->rx_packets = 0;
1039                 fwd_streams[sm_id]->tx_packets = 0;
1040                 fwd_streams[sm_id]->fwd_dropped = 0;
1041                 fwd_streams[sm_id]->rx_bad_ip_csum = 0;
1042                 fwd_streams[sm_id]->rx_bad_l4_csum = 0;
1043
1044 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1045                 memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
1046                        sizeof(fwd_streams[sm_id]->rx_burst_stats));
1047                 memset(&fwd_streams[sm_id]->tx_burst_stats, 0,
1048                        sizeof(fwd_streams[sm_id]->tx_burst_stats));
1049 #endif
1050 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1051                 fwd_streams[sm_id]->core_cycles = 0;
1052 #endif
1053         }
1054         if (with_tx_first) {
1055                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1056                 if (port_fwd_begin != NULL) {
1057                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1058                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1059                 }
1060                 while (with_tx_first--) {
1061                         launch_packet_forwarding(
1062                                         run_one_txonly_burst_on_core);
1063                         rte_eal_mp_wait_lcore();
1064                 }
1065                 port_fwd_end = tx_only_engine.port_fwd_end;
1066                 if (port_fwd_end != NULL) {
1067                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1068                                 (*port_fwd_end)(fwd_ports_ids[i]);
1069                 }
1070         }
1071         launch_packet_forwarding(start_pkt_forward_on_core);
1072 }
1073
1074 void
1075 stop_packet_forwarding(void)
1076 {
1077         struct rte_eth_stats stats;
1078         struct rte_port *port;
1079         port_fwd_end_t  port_fwd_end;
1080         int i;
1081         portid_t   pt_id;
1082         streamid_t sm_id;
1083         lcoreid_t  lc_id;
1084         uint64_t total_recv;
1085         uint64_t total_xmit;
1086         uint64_t total_rx_dropped;
1087         uint64_t total_tx_dropped;
1088         uint64_t total_rx_nombuf;
1089         uint64_t tx_dropped;
1090         uint64_t rx_bad_ip_csum;
1091         uint64_t rx_bad_l4_csum;
1092 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1093         uint64_t fwd_cycles;
1094 #endif
1095         static const char *acc_stats_border = "+++++++++++++++";
1096
1097         if (test_done) {
1098                 printf("Packet forwarding not started\n");
1099                 return;
1100         }
1101         printf("Telling cores to stop...");
1102         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1103                 fwd_lcores[lc_id]->stopped = 1;
1104         printf("\nWaiting for lcores to finish...\n");
1105         rte_eal_mp_wait_lcore();
1106         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1107         if (port_fwd_end != NULL) {
1108                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1109                         pt_id = fwd_ports_ids[i];
1110                         (*port_fwd_end)(pt_id);
1111                 }
1112         }
1113 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1114         fwd_cycles = 0;
1115 #endif
1116         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1117                 if (cur_fwd_config.nb_fwd_streams >
1118                     cur_fwd_config.nb_fwd_ports) {
1119                         fwd_stream_stats_display(sm_id);
1120                         ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL;
1121                         ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL;
1122                 } else {
1123                         ports[fwd_streams[sm_id]->tx_port].tx_stream =
1124                                 fwd_streams[sm_id];
1125                         ports[fwd_streams[sm_id]->rx_port].rx_stream =
1126                                 fwd_streams[sm_id];
1127                 }
1128                 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped;
1129                 tx_dropped = (uint64_t) (tx_dropped +
1130                                          fwd_streams[sm_id]->fwd_dropped);
1131                 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped;
1132
1133                 rx_bad_ip_csum =
1134                         ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum;
1135                 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum +
1136                                          fwd_streams[sm_id]->rx_bad_ip_csum);
1137                 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum =
1138                                                         rx_bad_ip_csum;
1139
1140                 rx_bad_l4_csum =
1141                         ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum;
1142                 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum +
1143                                          fwd_streams[sm_id]->rx_bad_l4_csum);
1144                 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum =
1145                                                         rx_bad_l4_csum;
1146
1147 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1148                 fwd_cycles = (uint64_t) (fwd_cycles +
1149                                          fwd_streams[sm_id]->core_cycles);
1150 #endif
1151         }
1152         total_recv = 0;
1153         total_xmit = 0;
1154         total_rx_dropped = 0;
1155         total_tx_dropped = 0;
1156         total_rx_nombuf  = 0;
1157         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1158                 pt_id = fwd_ports_ids[i];
1159
1160                 port = &ports[pt_id];
1161                 rte_eth_stats_get(pt_id, &stats);
1162                 stats.ipackets -= port->stats.ipackets;
1163                 port->stats.ipackets = 0;
1164                 stats.opackets -= port->stats.opackets;
1165                 port->stats.opackets = 0;
1166                 stats.ibytes   -= port->stats.ibytes;
1167                 port->stats.ibytes = 0;
1168                 stats.obytes   -= port->stats.obytes;
1169                 port->stats.obytes = 0;
1170                 stats.imissed  -= port->stats.imissed;
1171                 port->stats.imissed = 0;
1172                 stats.oerrors  -= port->stats.oerrors;
1173                 port->stats.oerrors = 0;
1174                 stats.rx_nombuf -= port->stats.rx_nombuf;
1175                 port->stats.rx_nombuf = 0;
1176
1177                 total_recv += stats.ipackets;
1178                 total_xmit += stats.opackets;
1179                 total_rx_dropped += stats.imissed;
1180                 total_tx_dropped += port->tx_dropped;
1181                 total_rx_nombuf  += stats.rx_nombuf;
1182
1183                 fwd_port_stats_display(pt_id, &stats);
1184         }
1185         printf("\n  %s Accumulated forward statistics for all ports"
1186                "%s\n",
1187                acc_stats_border, acc_stats_border);
1188         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1189                "%-"PRIu64"\n"
1190                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1191                "%-"PRIu64"\n",
1192                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1193                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1194         if (total_rx_nombuf > 0)
1195                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1196         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1197                "%s\n",
1198                acc_stats_border, acc_stats_border);
1199 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1200         if (total_recv > 0)
1201                 printf("\n  CPU cycles/packet=%u (total cycles="
1202                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1203                        (unsigned int)(fwd_cycles / total_recv),
1204                        fwd_cycles, total_recv);
1205 #endif
1206         printf("\nDone.\n");
1207         test_done = 1;
1208 }
1209
1210 void
1211 dev_set_link_up(portid_t pid)
1212 {
1213         if (rte_eth_dev_set_link_up((uint8_t)pid) < 0)
1214                 printf("\nSet link up fail.\n");
1215 }
1216
1217 void
1218 dev_set_link_down(portid_t pid)
1219 {
1220         if (rte_eth_dev_set_link_down((uint8_t)pid) < 0)
1221                 printf("\nSet link down fail.\n");
1222 }
1223
1224 static int
1225 all_ports_started(void)
1226 {
1227         portid_t pi;
1228         struct rte_port *port;
1229
1230         FOREACH_PORT(pi, ports) {
1231                 port = &ports[pi];
1232                 /* Check if there is a port which is not started */
1233                 if ((port->port_status != RTE_PORT_STARTED) &&
1234                         (port->slave_flag == 0))
1235                         return 0;
1236         }
1237
1238         /* No port is not started */
1239         return 1;
1240 }
1241
1242 int
1243 all_ports_stopped(void)
1244 {
1245         portid_t pi;
1246         struct rte_port *port;
1247
1248         FOREACH_PORT(pi, ports) {
1249                 port = &ports[pi];
1250                 if ((port->port_status != RTE_PORT_STOPPED) &&
1251                         (port->slave_flag == 0))
1252                         return 0;
1253         }
1254
1255         return 1;
1256 }
1257
1258 int
1259 port_is_started(portid_t port_id)
1260 {
1261         if (port_id_is_invalid(port_id, ENABLED_WARN))
1262                 return 0;
1263
1264         if (ports[port_id].port_status != RTE_PORT_STARTED)
1265                 return 0;
1266
1267         return 1;
1268 }
1269
1270 static int
1271 port_is_closed(portid_t port_id)
1272 {
1273         if (port_id_is_invalid(port_id, ENABLED_WARN))
1274                 return 0;
1275
1276         if (ports[port_id].port_status != RTE_PORT_CLOSED)
1277                 return 0;
1278
1279         return 1;
1280 }
1281
1282 int
1283 start_port(portid_t pid)
1284 {
1285         int diag, need_check_link_status = -1;
1286         portid_t pi;
1287         queueid_t qi;
1288         struct rte_port *port;
1289         struct ether_addr mac_addr;
1290
1291         if (port_id_is_invalid(pid, ENABLED_WARN))
1292                 return 0;
1293
1294         if(dcb_config)
1295                 dcb_test = 1;
1296         FOREACH_PORT(pi, ports) {
1297                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1298                         continue;
1299
1300                 need_check_link_status = 0;
1301                 port = &ports[pi];
1302                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1303                                                  RTE_PORT_HANDLING) == 0) {
1304                         printf("Port %d is now not stopped\n", pi);
1305                         continue;
1306                 }
1307
1308                 if (port->need_reconfig > 0) {
1309                         port->need_reconfig = 0;
1310
1311                         printf("Configuring Port %d (socket %u)\n", pi,
1312                                         port->socket_id);
1313                         /* configure port */
1314                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1315                                                 &(port->dev_conf));
1316                         if (diag != 0) {
1317                                 if (rte_atomic16_cmpset(&(port->port_status),
1318                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1319                                         printf("Port %d can not be set back "
1320                                                         "to stopped\n", pi);
1321                                 printf("Fail to configure port %d\n", pi);
1322                                 /* try to reconfigure port next time */
1323                                 port->need_reconfig = 1;
1324                                 return -1;
1325                         }
1326                 }
1327                 if (port->need_reconfig_queues > 0) {
1328                         port->need_reconfig_queues = 0;
1329                         /* setup tx queues */
1330                         for (qi = 0; qi < nb_txq; qi++) {
1331                                 if ((numa_support) &&
1332                                         (txring_numa[pi] != NUMA_NO_CONFIG))
1333                                         diag = rte_eth_tx_queue_setup(pi, qi,
1334                                                 nb_txd,txring_numa[pi],
1335                                                 &(port->tx_conf));
1336                                 else
1337                                         diag = rte_eth_tx_queue_setup(pi, qi,
1338                                                 nb_txd,port->socket_id,
1339                                                 &(port->tx_conf));
1340
1341                                 if (diag == 0)
1342                                         continue;
1343
1344                                 /* Fail to setup tx queue, return */
1345                                 if (rte_atomic16_cmpset(&(port->port_status),
1346                                                         RTE_PORT_HANDLING,
1347                                                         RTE_PORT_STOPPED) == 0)
1348                                         printf("Port %d can not be set back "
1349                                                         "to stopped\n", pi);
1350                                 printf("Fail to configure port %d tx queues\n", pi);
1351                                 /* try to reconfigure queues next time */
1352                                 port->need_reconfig_queues = 1;
1353                                 return -1;
1354                         }
1355                         /* setup rx queues */
1356                         for (qi = 0; qi < nb_rxq; qi++) {
1357                                 if ((numa_support) &&
1358                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
1359                                         struct rte_mempool * mp =
1360                                                 mbuf_pool_find(rxring_numa[pi]);
1361                                         if (mp == NULL) {
1362                                                 printf("Failed to setup RX queue:"
1363                                                         "No mempool allocation"
1364                                                         " on the socket %d\n",
1365                                                         rxring_numa[pi]);
1366                                                 return -1;
1367                                         }
1368
1369                                         diag = rte_eth_rx_queue_setup(pi, qi,
1370                                              nb_rxd,rxring_numa[pi],
1371                                              &(port->rx_conf),mp);
1372                                 } else {
1373                                         struct rte_mempool *mp =
1374                                                 mbuf_pool_find(port->socket_id);
1375                                         if (mp == NULL) {
1376                                                 printf("Failed to setup RX queue:"
1377                                                         "No mempool allocation"
1378                                                         " on the socket %d\n",
1379                                                         port->socket_id);
1380                                                 return -1;
1381                                         }
1382                                         diag = rte_eth_rx_queue_setup(pi, qi,
1383                                              nb_rxd,port->socket_id,
1384                                              &(port->rx_conf), mp);
1385                                 }
1386                                 if (diag == 0)
1387                                         continue;
1388
1389                                 /* Fail to setup rx queue, return */
1390                                 if (rte_atomic16_cmpset(&(port->port_status),
1391                                                         RTE_PORT_HANDLING,
1392                                                         RTE_PORT_STOPPED) == 0)
1393                                         printf("Port %d can not be set back "
1394                                                         "to stopped\n", pi);
1395                                 printf("Fail to configure port %d rx queues\n", pi);
1396                                 /* try to reconfigure queues next time */
1397                                 port->need_reconfig_queues = 1;
1398                                 return -1;
1399                         }
1400                 }
1401                 /* start port */
1402                 if (rte_eth_dev_start(pi) < 0) {
1403                         printf("Fail to start port %d\n", pi);
1404
1405                         /* Fail to setup rx queue, return */
1406                         if (rte_atomic16_cmpset(&(port->port_status),
1407                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1408                                 printf("Port %d can not be set back to "
1409                                                         "stopped\n", pi);
1410                         continue;
1411                 }
1412
1413                 if (rte_atomic16_cmpset(&(port->port_status),
1414                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
1415                         printf("Port %d can not be set into started\n", pi);
1416
1417                 rte_eth_macaddr_get(pi, &mac_addr);
1418                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
1419                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
1420                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
1421                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
1422
1423                 /* at least one port started, need checking link status */
1424                 need_check_link_status = 1;
1425         }
1426
1427         if (need_check_link_status == 1 && !no_link_check)
1428                 check_all_ports_link_status(RTE_PORT_ALL);
1429         else if (need_check_link_status == 0)
1430                 printf("Please stop the ports first\n");
1431
1432         printf("Done\n");
1433         return 0;
1434 }
1435
1436 void
1437 stop_port(portid_t pid)
1438 {
1439         portid_t pi;
1440         struct rte_port *port;
1441         int need_check_link_status = 0;
1442
1443         if (dcb_test) {
1444                 dcb_test = 0;
1445                 dcb_config = 0;
1446         }
1447
1448         if (port_id_is_invalid(pid, ENABLED_WARN))
1449                 return;
1450
1451         printf("Stopping ports...\n");
1452
1453         FOREACH_PORT(pi, ports) {
1454                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1455                         continue;
1456
1457                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1458                         printf("Please remove port %d from forwarding configuration.\n", pi);
1459                         continue;
1460                 }
1461
1462                 if (port_is_bonding_slave(pi)) {
1463                         printf("Please remove port %d from bonded device.\n", pi);
1464                         continue;
1465                 }
1466
1467                 port = &ports[pi];
1468                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
1469                                                 RTE_PORT_HANDLING) == 0)
1470                         continue;
1471
1472                 rte_eth_dev_stop(pi);
1473
1474                 if (rte_atomic16_cmpset(&(port->port_status),
1475                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1476                         printf("Port %d can not be set into stopped\n", pi);
1477                 need_check_link_status = 1;
1478         }
1479         if (need_check_link_status && !no_link_check)
1480                 check_all_ports_link_status(RTE_PORT_ALL);
1481
1482         printf("Done\n");
1483 }
1484
1485 void
1486 close_port(portid_t pid)
1487 {
1488         portid_t pi;
1489         struct rte_port *port;
1490
1491         if (port_id_is_invalid(pid, ENABLED_WARN))
1492                 return;
1493
1494         printf("Closing ports...\n");
1495
1496         FOREACH_PORT(pi, ports) {
1497                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1498                         continue;
1499
1500                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1501                         printf("Please remove port %d from forwarding configuration.\n", pi);
1502                         continue;
1503                 }
1504
1505                 if (port_is_bonding_slave(pi)) {
1506                         printf("Please remove port %d from bonded device.\n", pi);
1507                         continue;
1508                 }
1509
1510                 port = &ports[pi];
1511                 if (rte_atomic16_cmpset(&(port->port_status),
1512                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
1513                         printf("Port %d is already closed\n", pi);
1514                         continue;
1515                 }
1516
1517                 if (rte_atomic16_cmpset(&(port->port_status),
1518                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
1519                         printf("Port %d is now not stopped\n", pi);
1520                         continue;
1521                 }
1522
1523                 rte_eth_dev_close(pi);
1524
1525                 if (rte_atomic16_cmpset(&(port->port_status),
1526                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
1527                         printf("Port %d cannot be set to closed\n", pi);
1528         }
1529
1530         printf("Done\n");
1531 }
1532
1533 void
1534 attach_port(char *identifier)
1535 {
1536         portid_t pi = 0;
1537         unsigned int socket_id;
1538
1539         printf("Attaching a new port...\n");
1540
1541         if (identifier == NULL) {
1542                 printf("Invalid parameters are specified\n");
1543                 return;
1544         }
1545
1546         if (rte_eth_dev_attach(identifier, &pi))
1547                 return;
1548
1549         ports[pi].enabled = 1;
1550         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
1551         /* if socket_id is invalid, set to 0 */
1552         if (check_socket_id(socket_id) < 0)
1553                 socket_id = 0;
1554         reconfig(pi, socket_id);
1555         rte_eth_promiscuous_enable(pi);
1556
1557         nb_ports = rte_eth_dev_count();
1558
1559         ports[pi].port_status = RTE_PORT_STOPPED;
1560
1561         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
1562         printf("Done\n");
1563 }
1564
1565 void
1566 detach_port(uint8_t port_id)
1567 {
1568         char name[RTE_ETH_NAME_MAX_LEN];
1569
1570         printf("Detaching a port...\n");
1571
1572         if (!port_is_closed(port_id)) {
1573                 printf("Please close port first\n");
1574                 return;
1575         }
1576
1577         if (rte_eth_dev_detach(port_id, name))
1578                 return;
1579
1580         ports[port_id].enabled = 0;
1581         nb_ports = rte_eth_dev_count();
1582
1583         printf("Port '%s' is detached. Now total ports is %d\n",
1584                         name, nb_ports);
1585         printf("Done\n");
1586         return;
1587 }
1588
1589 void
1590 pmd_test_exit(void)
1591 {
1592         portid_t pt_id;
1593
1594         if (test_done == 0)
1595                 stop_packet_forwarding();
1596
1597         if (ports != NULL) {
1598                 no_link_check = 1;
1599                 FOREACH_PORT(pt_id, ports) {
1600                         printf("\nShutting down port %d...\n", pt_id);
1601                         fflush(stdout);
1602                         stop_port(pt_id);
1603                         close_port(pt_id);
1604                 }
1605         }
1606         printf("\nBye...\n");
1607 }
1608
1609 typedef void (*cmd_func_t)(void);
1610 struct pmd_test_command {
1611         const char *cmd_name;
1612         cmd_func_t cmd_func;
1613 };
1614
1615 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
1616
1617 /* Check the link status of all ports in up to 9s, and print them finally */
1618 static void
1619 check_all_ports_link_status(uint32_t port_mask)
1620 {
1621 #define CHECK_INTERVAL 100 /* 100ms */
1622 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
1623         uint8_t portid, count, all_ports_up, print_flag = 0;
1624         struct rte_eth_link link;
1625
1626         printf("Checking link statuses...\n");
1627         fflush(stdout);
1628         for (count = 0; count <= MAX_CHECK_TIME; count++) {
1629                 all_ports_up = 1;
1630                 FOREACH_PORT(portid, ports) {
1631                         if ((port_mask & (1 << portid)) == 0)
1632                                 continue;
1633                         memset(&link, 0, sizeof(link));
1634                         rte_eth_link_get_nowait(portid, &link);
1635                         /* print link status if flag set */
1636                         if (print_flag == 1) {
1637                                 if (link.link_status)
1638                                         printf("Port %d Link Up - speed %u "
1639                                                 "Mbps - %s\n", (uint8_t)portid,
1640                                                 (unsigned)link.link_speed,
1641                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
1642                                         ("full-duplex") : ("half-duplex\n"));
1643                                 else
1644                                         printf("Port %d Link Down\n",
1645                                                 (uint8_t)portid);
1646                                 continue;
1647                         }
1648                         /* clear all_ports_up flag if any link down */
1649                         if (link.link_status == ETH_LINK_DOWN) {
1650                                 all_ports_up = 0;
1651                                 break;
1652                         }
1653                 }
1654                 /* after finally printing all link status, get out */
1655                 if (print_flag == 1)
1656                         break;
1657
1658                 if (all_ports_up == 0) {
1659                         fflush(stdout);
1660                         rte_delay_ms(CHECK_INTERVAL);
1661                 }
1662
1663                 /* set the print_flag if all ports up or timeout */
1664                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
1665                         print_flag = 1;
1666                 }
1667         }
1668 }
1669
1670 static int
1671 set_tx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port)
1672 {
1673         uint16_t i;
1674         int diag;
1675         uint8_t mapping_found = 0;
1676
1677         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
1678                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
1679                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
1680                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
1681                                         tx_queue_stats_mappings[i].queue_id,
1682                                         tx_queue_stats_mappings[i].stats_counter_id);
1683                         if (diag != 0)
1684                                 return diag;
1685                         mapping_found = 1;
1686                 }
1687         }
1688         if (mapping_found)
1689                 port->tx_queue_stats_mapping_enabled = 1;
1690         return 0;
1691 }
1692
1693 static int
1694 set_rx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port)
1695 {
1696         uint16_t i;
1697         int diag;
1698         uint8_t mapping_found = 0;
1699
1700         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
1701                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
1702                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
1703                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
1704                                         rx_queue_stats_mappings[i].queue_id,
1705                                         rx_queue_stats_mappings[i].stats_counter_id);
1706                         if (diag != 0)
1707                                 return diag;
1708                         mapping_found = 1;
1709                 }
1710         }
1711         if (mapping_found)
1712                 port->rx_queue_stats_mapping_enabled = 1;
1713         return 0;
1714 }
1715
1716 static void
1717 map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port)
1718 {
1719         int diag = 0;
1720
1721         diag = set_tx_queue_stats_mapping_registers(pi, port);
1722         if (diag != 0) {
1723                 if (diag == -ENOTSUP) {
1724                         port->tx_queue_stats_mapping_enabled = 0;
1725                         printf("TX queue stats mapping not supported port id=%d\n", pi);
1726                 }
1727                 else
1728                         rte_exit(EXIT_FAILURE,
1729                                         "set_tx_queue_stats_mapping_registers "
1730                                         "failed for port id=%d diag=%d\n",
1731                                         pi, diag);
1732         }
1733
1734         diag = set_rx_queue_stats_mapping_registers(pi, port);
1735         if (diag != 0) {
1736                 if (diag == -ENOTSUP) {
1737                         port->rx_queue_stats_mapping_enabled = 0;
1738                         printf("RX queue stats mapping not supported port id=%d\n", pi);
1739                 }
1740                 else
1741                         rte_exit(EXIT_FAILURE,
1742                                         "set_rx_queue_stats_mapping_registers "
1743                                         "failed for port id=%d diag=%d\n",
1744                                         pi, diag);
1745         }
1746 }
1747
1748 static void
1749 rxtx_port_config(struct rte_port *port)
1750 {
1751         port->rx_conf = port->dev_info.default_rxconf;
1752         port->tx_conf = port->dev_info.default_txconf;
1753
1754         /* Check if any RX/TX parameters have been passed */
1755         if (rx_pthresh != RTE_PMD_PARAM_UNSET)
1756                 port->rx_conf.rx_thresh.pthresh = rx_pthresh;
1757
1758         if (rx_hthresh != RTE_PMD_PARAM_UNSET)
1759                 port->rx_conf.rx_thresh.hthresh = rx_hthresh;
1760
1761         if (rx_wthresh != RTE_PMD_PARAM_UNSET)
1762                 port->rx_conf.rx_thresh.wthresh = rx_wthresh;
1763
1764         if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
1765                 port->rx_conf.rx_free_thresh = rx_free_thresh;
1766
1767         if (rx_drop_en != RTE_PMD_PARAM_UNSET)
1768                 port->rx_conf.rx_drop_en = rx_drop_en;
1769
1770         if (tx_pthresh != RTE_PMD_PARAM_UNSET)
1771                 port->tx_conf.tx_thresh.pthresh = tx_pthresh;
1772
1773         if (tx_hthresh != RTE_PMD_PARAM_UNSET)
1774                 port->tx_conf.tx_thresh.hthresh = tx_hthresh;
1775
1776         if (tx_wthresh != RTE_PMD_PARAM_UNSET)
1777                 port->tx_conf.tx_thresh.wthresh = tx_wthresh;
1778
1779         if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
1780                 port->tx_conf.tx_rs_thresh = tx_rs_thresh;
1781
1782         if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
1783                 port->tx_conf.tx_free_thresh = tx_free_thresh;
1784
1785         if (txq_flags != RTE_PMD_PARAM_UNSET)
1786                 port->tx_conf.txq_flags = txq_flags;
1787 }
1788
1789 void
1790 init_port_config(void)
1791 {
1792         portid_t pid;
1793         struct rte_port *port;
1794
1795         FOREACH_PORT(pid, ports) {
1796                 port = &ports[pid];
1797                 port->dev_conf.rxmode = rx_mode;
1798                 port->dev_conf.fdir_conf = fdir_conf;
1799                 if (nb_rxq > 1) {
1800                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1801                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = rss_hf;
1802                 } else {
1803                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1804                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
1805                 }
1806
1807                 if (port->dcb_flag == 0 && port->dev_info.max_vfs == 0) {
1808                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
1809                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
1810                         else
1811                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
1812                 }
1813
1814                 if (port->dev_info.max_vfs != 0) {
1815                         if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
1816                                 port->dev_conf.rxmode.mq_mode =
1817                                         ETH_MQ_RX_VMDQ_RSS;
1818                         else
1819                                 port->dev_conf.rxmode.mq_mode =
1820                                         ETH_MQ_RX_NONE;
1821
1822                         port->dev_conf.txmode.mq_mode = ETH_MQ_TX_NONE;
1823                 }
1824
1825                 rxtx_port_config(port);
1826
1827                 rte_eth_macaddr_get(pid, &port->eth_addr);
1828
1829                 map_port_queue_stats_mapping_registers(pid, port);
1830 #ifdef RTE_NIC_BYPASS
1831                 rte_eth_dev_bypass_init(pid);
1832 #endif
1833         }
1834 }
1835
1836 void set_port_slave_flag(portid_t slave_pid)
1837 {
1838         struct rte_port *port;
1839
1840         port = &ports[slave_pid];
1841         port->slave_flag = 1;
1842 }
1843
1844 void clear_port_slave_flag(portid_t slave_pid)
1845 {
1846         struct rte_port *port;
1847
1848         port = &ports[slave_pid];
1849         port->slave_flag = 0;
1850 }
1851
1852 uint8_t port_is_bonding_slave(portid_t slave_pid)
1853 {
1854         struct rte_port *port;
1855
1856         port = &ports[slave_pid];
1857         return port->slave_flag;
1858 }
1859
1860 const uint16_t vlan_tags[] = {
1861                 0,  1,  2,  3,  4,  5,  6,  7,
1862                 8,  9, 10, 11,  12, 13, 14, 15,
1863                 16, 17, 18, 19, 20, 21, 22, 23,
1864                 24, 25, 26, 27, 28, 29, 30, 31
1865 };
1866
1867 static  int
1868 get_eth_dcb_conf(struct rte_eth_conf *eth_conf,
1869                  enum dcb_mode_enable dcb_mode,
1870                  enum rte_eth_nb_tcs num_tcs,
1871                  uint8_t pfc_en)
1872 {
1873         uint8_t i;
1874
1875         /*
1876          * Builds up the correct configuration for dcb+vt based on the vlan tags array
1877          * given above, and the number of traffic classes available for use.
1878          */
1879         if (dcb_mode == DCB_VT_ENABLED) {
1880                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
1881                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
1882                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
1883                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
1884
1885                 /* VMDQ+DCB RX and TX configrations */
1886                 vmdq_rx_conf->enable_default_pool = 0;
1887                 vmdq_rx_conf->default_pool = 0;
1888                 vmdq_rx_conf->nb_queue_pools =
1889                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
1890                 vmdq_tx_conf->nb_queue_pools =
1891                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
1892
1893                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
1894                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
1895                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
1896                         vmdq_rx_conf->pool_map[i].pools =
1897                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
1898                 }
1899                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
1900                         vmdq_rx_conf->dcb_tc[i] = i;
1901                         vmdq_tx_conf->dcb_tc[i] = i;
1902                 }
1903
1904                 /* set DCB mode of RX and TX of multiple queues */
1905                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
1906                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
1907         } else {
1908                 struct rte_eth_dcb_rx_conf *rx_conf =
1909                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
1910                 struct rte_eth_dcb_tx_conf *tx_conf =
1911                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
1912
1913                 rx_conf->nb_tcs = num_tcs;
1914                 tx_conf->nb_tcs = num_tcs;
1915
1916                 for (i = 0; i < num_tcs; i++) {
1917                         rx_conf->dcb_tc[i] = i;
1918                         tx_conf->dcb_tc[i] = i;
1919                 }
1920                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
1921                 eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf;
1922                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
1923         }
1924
1925         if (pfc_en)
1926                 eth_conf->dcb_capability_en =
1927                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
1928         else
1929                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
1930
1931         return 0;
1932 }
1933
1934 int
1935 init_port_dcb_config(portid_t pid,
1936                      enum dcb_mode_enable dcb_mode,
1937                      enum rte_eth_nb_tcs num_tcs,
1938                      uint8_t pfc_en)
1939 {
1940         struct rte_eth_conf port_conf;
1941         struct rte_eth_dev_info dev_info;
1942         struct rte_port *rte_port;
1943         int retval;
1944         uint16_t i;
1945
1946         rte_eth_dev_info_get(pid, &dev_info);
1947
1948         /* If dev_info.vmdq_pool_base is greater than 0,
1949          * the queue id of vmdq pools is started after pf queues.
1950          */
1951         if (dcb_mode == DCB_VT_ENABLED && dev_info.vmdq_pool_base > 0) {
1952                 printf("VMDQ_DCB multi-queue mode is nonsensical"
1953                         " for port %d.", pid);
1954                 return -1;
1955         }
1956
1957         /* Assume the ports in testpmd have the same dcb capability
1958          * and has the same number of rxq and txq in dcb mode
1959          */
1960         if (dcb_mode == DCB_VT_ENABLED) {
1961                 nb_rxq = dev_info.max_rx_queues;
1962                 nb_txq = dev_info.max_tx_queues;
1963         } else {
1964                 /*if vt is disabled, use all pf queues */
1965                 if (dev_info.vmdq_pool_base == 0) {
1966                         nb_rxq = dev_info.max_rx_queues;
1967                         nb_txq = dev_info.max_tx_queues;
1968                 } else {
1969                         nb_rxq = (queueid_t)num_tcs;
1970                         nb_txq = (queueid_t)num_tcs;
1971
1972                 }
1973         }
1974         rx_free_thresh = 64;
1975
1976         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
1977         /* Enter DCB configuration status */
1978         dcb_config = 1;
1979
1980         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
1981         retval = get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en);
1982         if (retval < 0)
1983                 return retval;
1984
1985         rte_port = &ports[pid];
1986         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
1987
1988         rxtx_port_config(rte_port);
1989         /* VLAN filter */
1990         rte_port->dev_conf.rxmode.hw_vlan_filter = 1;
1991         for (i = 0; i < RTE_DIM(vlan_tags); i++)
1992                 rx_vft_set(pid, vlan_tags[i], 1);
1993
1994         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
1995         map_port_queue_stats_mapping_registers(pid, rte_port);
1996
1997         rte_port->dcb_flag = 1;
1998
1999         return 0;
2000 }
2001
2002 static void
2003 init_port(void)
2004 {
2005         portid_t pid;
2006
2007         /* Configuration of Ethernet ports. */
2008         ports = rte_zmalloc("testpmd: ports",
2009                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
2010                             RTE_CACHE_LINE_SIZE);
2011         if (ports == NULL) {
2012                 rte_exit(EXIT_FAILURE,
2013                                 "rte_zmalloc(%d struct rte_port) failed\n",
2014                                 RTE_MAX_ETHPORTS);
2015         }
2016
2017         /* enabled allocated ports */
2018         for (pid = 0; pid < nb_ports; pid++)
2019                 ports[pid].enabled = 1;
2020 }
2021
2022 static void
2023 force_quit(void)
2024 {
2025         pmd_test_exit();
2026         prompt_exit();
2027 }
2028
2029 static void
2030 signal_handler(int signum)
2031 {
2032         if (signum == SIGINT || signum == SIGTERM) {
2033                 printf("\nSignal %d received, preparing to exit...\n",
2034                                 signum);
2035                 force_quit();
2036                 /* exit with the expected status */
2037                 signal(signum, SIG_DFL);
2038                 kill(getpid(), signum);
2039         }
2040 }
2041
2042 int
2043 main(int argc, char** argv)
2044 {
2045         int  diag;
2046         uint8_t port_id;
2047
2048         signal(SIGINT, signal_handler);
2049         signal(SIGTERM, signal_handler);
2050
2051         diag = rte_eal_init(argc, argv);
2052         if (diag < 0)
2053                 rte_panic("Cannot init EAL\n");
2054
2055         nb_ports = (portid_t) rte_eth_dev_count();
2056         if (nb_ports == 0)
2057                 RTE_LOG(WARNING, EAL, "No probed ethernet devices\n");
2058
2059         /* allocate port structures, and init them */
2060         init_port();
2061
2062         set_def_fwd_config();
2063         if (nb_lcores == 0)
2064                 rte_panic("Empty set of forwarding logical cores - check the "
2065                           "core mask supplied in the command parameters\n");
2066
2067         argc -= diag;
2068         argv += diag;
2069         if (argc > 1)
2070                 launch_args_parse(argc, argv);
2071
2072         if (!nb_rxq && !nb_txq)
2073                 printf("Warning: Either rx or tx queues should be non-zero\n");
2074
2075         if (nb_rxq > 1 && nb_rxq > nb_txq)
2076                 printf("Warning: nb_rxq=%d enables RSS configuration, "
2077                        "but nb_txq=%d will prevent to fully test it.\n",
2078                        nb_rxq, nb_txq);
2079
2080         init_config();
2081         if (start_port(RTE_PORT_ALL) != 0)
2082                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
2083
2084         /* set all ports to promiscuous mode by default */
2085         FOREACH_PORT(port_id, ports)
2086                 rte_eth_promiscuous_enable(port_id);
2087
2088 #ifdef RTE_LIBRTE_CMDLINE
2089         if (interactive == 1) {
2090                 if (auto_start) {
2091                         printf("Start automatic packet forwarding\n");
2092                         start_packet_forwarding(0);
2093                 }
2094                 prompt();
2095         } else
2096 #endif
2097         {
2098                 char c;
2099                 int rc;
2100
2101                 printf("No commandline core given, start packet forwarding\n");
2102                 start_packet_forwarding(0);
2103                 printf("Press enter to exit\n");
2104                 rc = read(0, &c, 1);
2105                 pmd_test_exit();
2106                 if (rc < 0)
2107                         return 1;
2108         }
2109
2110         return 0;
2111 }