app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATE
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 uint16_t verbose_level = 0; /**< Silent by default. */
  67 int testpmd_logtype; /**< Log type for testpmd logs */
  68
  69 /* use master core for command line ? */
  70 uint8_t interactive = 0;
  71 uint8_t auto_start = 0;
  72 uint8_t tx_first;
  73 char cmdline_filename[PATH_MAX] = {0};
  74
  75 /*
  76  * NUMA support configuration.
  77  * When set, the NUMA support attempts to dispatch the allocation of the
  78  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  79  * probed ports among the CPU sockets 0 and 1.
  80  * Otherwise, all memory is allocated from CPU socket 0.
  81  */
  82 uint8_t numa_support = 1; /**< numa enabled by default */
  83
  84 /*
  85  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
  86  * not configured.
  87  */
  88 uint8_t socket_num = UMA_NO_CONFIG;
  89
  90 /*
  91  * Use ANONYMOUS mapped memory (might be not physically continuous) for mbufs.
  92  */
  93 uint8_t mp_anon = 0;
  94
  95 /*
  96  * Store specified sockets on which memory pool to be used by ports
  97  * is allocated.
  98  */
  99 uint8_t port_numa[RTE_MAX_ETHPORTS];
 100
 101 /*
 102  * Store specified sockets on which RX ring to be used by ports
 103  * is allocated.
 104  */
 105 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 106
 107 /*
 108  * Store specified sockets on which TX ring to be used by ports
 109  * is allocated.
 110  */
 111 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 112
 113 /*
 114  * Record the Ethernet address of peer target ports to which packets are
 115  * forwarded.
 116  * Must be instantiated with the ethernet addresses of peer traffic generator
 117  * ports.
 118  */
 119 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 120 portid_t nb_peer_eth_addrs = 0;
 121
 122 /*
 123  * Probed Target Environment.
 124  */
 125 struct rte_port *ports;        /**< For all probed ethernet ports. */
 126 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 127 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 128 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 129
 130 /*
 131  * Test Forwarding Configuration.
 132  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 133  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 134  */
 135 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 136 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 137 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 138 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 139
 140 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 141 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 142
 143 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 144 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 145
 146 /*
 147  * Forwarding engines.
 148  */
 149 struct fwd_engine * fwd_engines[] = {
 150         &io_fwd_engine,
 151         &mac_fwd_engine,
 152         &mac_swap_engine,
 153         &flow_gen_engine,
 154         &rx_only_engine,
 155         &tx_only_engine,
 156         &csum_fwd_engine,
 157         &icmp_echo_engine,
 158 #if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED
 159         &softnic_tm_engine,
 160         &softnic_tm_bypass_engine,
 161 #endif
 162 #ifdef RTE_LIBRTE_IEEE1588
 163         &ieee1588_fwd_engine,
 164 #endif
 165         NULL,
 166 };
 167
 168 struct fwd_config cur_fwd_config;
 169 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 170 uint32_t retry_enabled;
 171 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 172 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 173
 174 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 175 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 176                                       * specified on command-line. */
 177 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 178
 179 /*
 180  * In container, it cannot terminate the process which running with 'stats-period'
 181  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 182  */
 183 uint8_t f_quit;
 184
 185 /*
 186  * Configuration of packet segments used by the "txonly" processing engine.
 187  */
 188 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 189 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 190         TXONLY_DEF_PACKET_LEN,
 191 };
 192 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 193
 194 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 195 /**< Split policy for packets to TX. */
 196
 197 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 198 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 199
 200 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 201 uint8_t dcb_config = 0;
 202
 203 /* Whether the dcb is in testing status */
 204 uint8_t dcb_test = 0;
 205
 206 /*
 207  * Configurable number of RX/TX queues.
 208  */
 209 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 210 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 211
 212 /*
 213  * Configurable number of RX/TX ring descriptors.
 214  * Defaults are supplied by drivers via ethdev.
 215  */
 216 #define RTE_TEST_RX_DESC_DEFAULT 0
 217 #define RTE_TEST_TX_DESC_DEFAULT 0
 218 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 219 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 220
 221 #define RTE_PMD_PARAM_UNSET -1
 222 /*
 223  * Configurable values of RX and TX ring threshold registers.
 224  */
 225
 226 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 227 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 228 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 229
 230 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 231 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 232 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 233
 234 /*
 235  * Configurable value of RX free threshold.
 236  */
 237 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 238
 239 /*
 240  * Configurable value of RX drop enable.
 241  */
 242 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 243
 244 /*
 245  * Configurable value of TX free threshold.
 246  */
 247 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 248
 249 /*
 250  * Configurable value of TX RS bit threshold.
 251  */
 252 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 253
 254 /*
 255  * Receive Side Scaling (RSS) configuration.
 256  */
 257 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 258
 259 /*
 260  * Port topology configuration
 261  */
 262 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 263
 264 /*
 265  * Avoids to flush all the RX streams before starts forwarding.
 266  */
 267 uint8_t no_flush_rx = 0; /* flush by default */
 268
 269 /*
 270  * Flow API isolated mode.
 271  */
 272 uint8_t flow_isolate_all;
 273
 274 /*
 275  * Avoids to check link status when starting/stopping a port.
 276  */
 277 uint8_t no_link_check = 0; /* check by default */
 278
 279 /*
 280  * Enable link status change notification
 281  */
 282 uint8_t lsc_interrupt = 1; /* enabled by default */
 283
 284 /*
 285  * Enable device removal notification.
 286  */
 287 uint8_t rmv_interrupt = 1; /* enabled by default */
 288
 289 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 290
 291 /*
 292  * Display or mask ether events
 293  * Default to all events except VF_MBOX
 294  */
 295 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 296                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 297                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 298                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 299                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 300                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 301                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 302 /*
 303  * Decide if all memory are locked for performance.
 304  */
 305 int do_mlockall = 0;
 306
 307 /*
 308  * NIC bypass mode configuration options.
 309  */
 310
 311 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 312 /* The NIC bypass watchdog timeout. */
 313 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 314 #endif
 315
 316
 317 #ifdef RTE_LIBRTE_LATENCY_STATS
 318
 319 /*
 320  * Set when latency stats is enabled in the commandline
 321  */
 322 uint8_t latencystats_enabled;
 323
 324 /*
 325  * Lcore ID to serive latency statistics.
 326  */
 327 lcoreid_t latencystats_lcore_id = -1;
 328
 329 #endif
 330
 331 /*
 332  * Ethernet device configuration.
 333  */
 334 struct rte_eth_rxmode rx_mode = {
 335         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 336         .offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 337         .ignore_offload_bitfield = 1,
 338 };
 339
 340 struct rte_eth_txmode tx_mode = {
 341         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 342 };
 343
 344 struct rte_fdir_conf fdir_conf = {
 345         .mode = RTE_FDIR_MODE_NONE,
 346         .pballoc = RTE_FDIR_PBALLOC_64K,
 347         .status = RTE_FDIR_REPORT_STATUS,
 348         .mask = {
 349                 .vlan_tci_mask = 0x0,
 350                 .ipv4_mask     = {
 351                         .src_ip = 0xFFFFFFFF,
 352                         .dst_ip = 0xFFFFFFFF,
 353                 },
 354                 .ipv6_mask     = {
 355                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 356                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 357                 },
 358                 .src_port_mask = 0xFFFF,
 359                 .dst_port_mask = 0xFFFF,
 360                 .mac_addr_byte_mask = 0xFF,
 361                 .tunnel_type_mask = 1,
 362                 .tunnel_id_mask = 0xFFFFFFFF,
 363         },
 364         .drop_queue = 127,
 365 };
 366
 367 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 368
 369 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 370 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 371
 372 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 373 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 374
 375 uint16_t nb_tx_queue_stats_mappings = 0;
 376 uint16_t nb_rx_queue_stats_mappings = 0;
 377
 378 /*
 379  * Display zero values by default for xstats
 380  */
 381 uint8_t xstats_hide_zero;
 382
 383 unsigned int num_sockets = 0;
 384 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 385
 386 #ifdef RTE_LIBRTE_BITRATE
 387 /* Bitrate statistics */
 388 struct rte_stats_bitrates *bitrate_data;
 389 lcoreid_t bitrate_lcore_id;
 390 uint8_t bitrate_enabled;
 391 #endif
 392
 393 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 394 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 395
 396 /* Forward function declarations */
 397 static void map_port_queue_stats_mapping_registers(portid_t pi,
 398                                                    struct rte_port *port);
 399 static void check_all_ports_link_status(uint32_t port_mask);
 400 static int eth_event_callback(portid_t port_id,
 401                               enum rte_eth_event_type type,
 402                               void *param, void *ret_param);
 403 static void eth_dev_event_callback(char *device_name,
 404                                 enum rte_dev_event_type type,
 405                                 void *param);
 406 static int eth_dev_event_callback_register(void);
 407 static int eth_dev_event_callback_unregister(void);
 408
 409
 410 /*
 411  * Check if all the ports are started.
 412  * If yes, return positive value. If not, return zero.
 413  */
 414 static int all_ports_started(void);
 415
 416 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 417 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN;
 418
 419 /*
 420  * Helper function to check if socket is already discovered.
 421  * If yes, return positive value. If not, return zero.
 422  */
 423 int
 424 new_socket_id(unsigned int socket_id)
 425 {
 426         unsigned int i;
 427
 428         for (i = 0; i < num_sockets; i++) {
 429                 if (socket_ids[i] == socket_id)
 430                         return 0;
 431         }
 432         return 1;
 433 }
 434
 435 /*
 436  * Setup default configuration.
 437  */
 438 static void
 439 set_default_fwd_lcores_config(void)
 440 {
 441         unsigned int i;
 442         unsigned int nb_lc;
 443         unsigned int sock_num;
 444
 445         nb_lc = 0;
 446         for (i = 0; i < RTE_MAX_LCORE; i++) {
 447                 sock_num = rte_lcore_to_socket_id(i);
 448                 if (new_socket_id(sock_num)) {
 449                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 450                                 rte_exit(EXIT_FAILURE,
 451                                          "Total sockets greater than %u\n",
 452                                          RTE_MAX_NUMA_NODES);
 453                         }
 454                         socket_ids[num_sockets++] = sock_num;
 455                 }
 456                 if (!rte_lcore_is_enabled(i))
 457                         continue;
 458                 if (i == rte_get_master_lcore())
 459                         continue;
 460                 fwd_lcores_cpuids[nb_lc++] = i;
 461         }
 462         nb_lcores = (lcoreid_t) nb_lc;
 463         nb_cfg_lcores = nb_lcores;
 464         nb_fwd_lcores = 1;
 465 }
 466
 467 static void
 468 set_def_peer_eth_addrs(void)
 469 {
 470         portid_t i;
 471
 472         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 473                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 474                 peer_eth_addrs[i].addr_bytes[5] = i;
 475         }
 476 }
 477
 478 static void
 479 set_default_fwd_ports_config(void)
 480 {
 481         portid_t pt_id;
 482         int i = 0;
 483
 484         RTE_ETH_FOREACH_DEV(pt_id)
 485                 fwd_ports_ids[i++] = pt_id;
 486
 487         nb_cfg_ports = nb_ports;
 488         nb_fwd_ports = nb_ports;
 489 }
 490
 491 void
 492 set_def_fwd_config(void)
 493 {
 494         set_default_fwd_lcores_config();
 495         set_def_peer_eth_addrs();
 496         set_default_fwd_ports_config();
 497 }
 498
 499 /*
 500  * Configuration initialisation done once at init time.
 501  */
 502 static void
 503 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 504                  unsigned int socket_id)
 505 {
 506         char pool_name[RTE_MEMPOOL_NAMESIZE];
 507         struct rte_mempool *rte_mp = NULL;
 508         uint32_t mb_size;
 509
 510         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 511         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 512
 513         TESTPMD_LOG(INFO,
 514                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 515                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 516
 517         if (mp_anon != 0) {
 518                 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 519                         mb_size, (unsigned) mb_mempool_cache,
 520                         sizeof(struct rte_pktmbuf_pool_private),
 521                         socket_id, 0);
 522                 if (rte_mp == NULL)
 523                         goto err;
 524
 525                 if (rte_mempool_populate_anon(rte_mp) == 0) {
 526                         rte_mempool_free(rte_mp);
 527                         rte_mp = NULL;
 528                         goto err;
 529                 }
 530                 rte_pktmbuf_pool_init(rte_mp, NULL);
 531                 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 532         } else {
 533                 /* wrapper to rte_mempool_create() */
 534                 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 535                                 rte_mbuf_best_mempool_ops());
 536                 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 537                         mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 538         }
 539
 540 err:
 541         if (rte_mp == NULL) {
 542                 rte_exit(EXIT_FAILURE,
 543                         "Creation of mbuf pool for socket %u failed: %s\n",
 544                         socket_id, rte_strerror(rte_errno));
 545         } else if (verbose_level > 0) {
 546                 rte_mempool_dump(stdout, rte_mp);
 547         }
 548 }
 549
 550 /*
 551  * Check given socket id is valid or not with NUMA mode,
 552  * if valid, return 0, else return -1
 553  */
 554 static int
 555 check_socket_id(const unsigned int socket_id)
 556 {
 557         static int warning_once = 0;
 558
 559         if (new_socket_id(socket_id)) {
 560                 if (!warning_once && numa_support)
 561                         printf("Warning: NUMA should be configured manually by"
 562                                " using --port-numa-config and"
 563                                " --ring-numa-config parameters along with"
 564                                " --numa.\n");
 565                 warning_once = 1;
 566                 return -1;
 567         }
 568         return 0;
 569 }
 570
 571 /*
 572  * Get the allowed maximum number of RX queues.
 573  * *pid return the port id which has minimal value of
 574  * max_rx_queues in all ports.
 575  */
 576 queueid_t
 577 get_allowed_max_nb_rxq(portid_t *pid)
 578 {
 579         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
 580         portid_t pi;
 581         struct rte_eth_dev_info dev_info;
 582
 583         RTE_ETH_FOREACH_DEV(pi) {
 584                 rte_eth_dev_info_get(pi, &dev_info);
 585                 if (dev_info.max_rx_queues < allowed_max_rxq) {
 586                         allowed_max_rxq = dev_info.max_rx_queues;
 587                         *pid = pi;
 588                 }
 589         }
 590         return allowed_max_rxq;
 591 }
 592
 593 /*
 594  * Check input rxq is valid or not.
 595  * If input rxq is not greater than any of maximum number
 596  * of RX queues of all ports, it is valid.
 597  * if valid, return 0, else return -1
 598  */
 599 int
 600 check_nb_rxq(queueid_t rxq)
 601 {
 602         queueid_t allowed_max_rxq;
 603         portid_t pid = 0;
 604
 605         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
 606         if (rxq > allowed_max_rxq) {
 607                 printf("Fail: input rxq (%u) can't be greater "
 608                        "than max_rx_queues (%u) of port %u\n",
 609                        rxq,
 610                        allowed_max_rxq,
 611                        pid);
 612                 return -1;
 613         }
 614         return 0;
 615 }
 616
 617 /*
 618  * Get the allowed maximum number of TX queues.
 619  * *pid return the port id which has minimal value of
 620  * max_tx_queues in all ports.
 621  */
 622 queueid_t
 623 get_allowed_max_nb_txq(portid_t *pid)
 624 {
 625         queueid_t allowed_max_txq = MAX_QUEUE_ID;
 626         portid_t pi;
 627         struct rte_eth_dev_info dev_info;
 628
 629         RTE_ETH_FOREACH_DEV(pi) {
 630                 rte_eth_dev_info_get(pi, &dev_info);
 631                 if (dev_info.max_tx_queues < allowed_max_txq) {
 632                         allowed_max_txq = dev_info.max_tx_queues;
 633                         *pid = pi;
 634                 }
 635         }
 636         return allowed_max_txq;
 637 }
 638
 639 /*
 640  * Check input txq is valid or not.
 641  * If input txq is not greater than any of maximum number
 642  * of TX queues of all ports, it is valid.
 643  * if valid, return 0, else return -1
 644  */
 645 int
 646 check_nb_txq(queueid_t txq)
 647 {
 648         queueid_t allowed_max_txq;
 649         portid_t pid = 0;
 650
 651         allowed_max_txq = get_allowed_max_nb_txq(&pid);
 652         if (txq > allowed_max_txq) {
 653                 printf("Fail: input txq (%u) can't be greater "
 654                        "than max_tx_queues (%u) of port %u\n",
 655                        txq,
 656                        allowed_max_txq,
 657                        pid);
 658                 return -1;
 659         }
 660         return 0;
 661 }
 662
 663 static void
 664 init_config(void)
 665 {
 666         portid_t pid;
 667         struct rte_port *port;
 668         struct rte_mempool *mbp;
 669         unsigned int nb_mbuf_per_pool;
 670         lcoreid_t  lc_id;
 671         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
 672         struct rte_gro_param gro_param;
 673         uint32_t gso_types;
 674
 675         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
 676
 677         if (numa_support) {
 678                 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
 679                 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
 680                 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
 681         }
 682
 683         /* Configuration of logical cores. */
 684         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
 685                                 sizeof(struct fwd_lcore *) * nb_lcores,
 686                                 RTE_CACHE_LINE_SIZE);
 687         if (fwd_lcores == NULL) {
 688                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
 689                                                         "failed\n", nb_lcores);
 690         }
 691         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 692                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
 693                                                sizeof(struct fwd_lcore),
 694                                                RTE_CACHE_LINE_SIZE);
 695                 if (fwd_lcores[lc_id] == NULL) {
 696                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
 697                                                                 "failed\n");
 698                 }
 699                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
 700         }
 701
 702         RTE_ETH_FOREACH_DEV(pid) {
 703                 port = &ports[pid];
 704                 /* Apply default TxRx configuration for all ports */
 705                 port->dev_conf.txmode = tx_mode;
 706                 port->dev_conf.rxmode = rx_mode;
 707                 rte_eth_dev_info_get(pid, &port->dev_info);
 708                 if (!(port->dev_info.tx_offload_capa &
 709                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
 710                         port->dev_conf.txmode.offloads &=
 711                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 712                 if (numa_support) {
 713                         if (port_numa[pid] != NUMA_NO_CONFIG)
 714                                 port_per_socket[port_numa[pid]]++;
 715                         else {
 716                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
 717
 718                                 /* if socket_id is invalid, set to 0 */
 719                                 if (check_socket_id(socket_id) < 0)
 720                                         socket_id = 0;
 721                                 port_per_socket[socket_id]++;
 722                         }
 723                 }
 724
 725                 /* set flag to initialize port/queue */
 726                 port->need_reconfig = 1;
 727                 port->need_reconfig_queues = 1;
 728         }
 729
 730         /*
 731          * Create pools of mbuf.
 732          * If NUMA support is disabled, create a single pool of mbuf in
 733          * socket 0 memory by default.
 734          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
 735          *
 736          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
 737          * nb_txd can be configured at run time.
 738          */
 739         if (param_total_num_mbufs)
 740                 nb_mbuf_per_pool = param_total_num_mbufs;
 741         else {
 742                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
 743                         (nb_lcores * mb_mempool_cache) +
 744                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
 745                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
 746         }
 747
 748         if (numa_support) {
 749                 uint8_t i;
 750
 751                 for (i = 0; i < num_sockets; i++)
 752                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
 753                                          socket_ids[i]);
 754         } else {
 755                 if (socket_num == UMA_NO_CONFIG)
 756                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0);
 757                 else
 758                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
 759                                                  socket_num);
 760         }
 761
 762         init_port_config();
 763
 764         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
 765                 DEV_TX_OFFLOAD_GRE_TNL_TSO;
 766         /*
 767          * Records which Mbuf pool to use by each logical core, if needed.
 768          */
 769         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 770                 mbp = mbuf_pool_find(
 771                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
 772
 773                 if (mbp == NULL)
 774                         mbp = mbuf_pool_find(0);
 775                 fwd_lcores[lc_id]->mbp = mbp;
 776                 /* initialize GSO context */
 777                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
 778                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
 779                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
 780                 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN -
 781                         ETHER_CRC_LEN;
 782                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
 783         }
 784
 785         /* Configuration of packet forwarding streams. */
 786         if (init_fwd_streams() < 0)
 787                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 788
 789         fwd_config_setup();
 790
 791         /* create a gro context for each lcore */
 792         gro_param.gro_types = RTE_GRO_TCP_IPV4;
 793         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
 794         gro_param.max_item_per_flow = MAX_PKT_BURST;
 795         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 796                 gro_param.socket_id = rte_lcore_to_socket_id(
 797                                 fwd_lcores_cpuids[lc_id]);
 798                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
 799                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
 800                         rte_exit(EXIT_FAILURE,
 801                                         "rte_gro_ctx_create() failed\n");
 802                 }
 803         }
 804 }
 805
 806
 807 void
 808 reconfig(portid_t new_port_id, unsigned socket_id)
 809 {
 810         struct rte_port *port;
 811
 812         /* Reconfiguration of Ethernet ports. */
 813         port = &ports[new_port_id];
 814         rte_eth_dev_info_get(new_port_id, &port->dev_info);
 815
 816         /* set flag to initialize port/queue */
 817         port->need_reconfig = 1;
 818         port->need_reconfig_queues = 1;
 819         port->socket_id = socket_id;
 820
 821         init_port_config();
 822 }
 823
 824
 825 int
 826 init_fwd_streams(void)
 827 {
 828         portid_t pid;
 829         struct rte_port *port;
 830         streamid_t sm_id, nb_fwd_streams_new;
 831         queueid_t q;
 832
 833         /* set socket id according to numa or not */
 834         RTE_ETH_FOREACH_DEV(pid) {
 835                 port = &ports[pid];
 836                 if (nb_rxq > port->dev_info.max_rx_queues) {
 837                         printf("Fail: nb_rxq(%d) is greater than "
 838                                 "max_rx_queues(%d)\n", nb_rxq,
 839                                 port->dev_info.max_rx_queues);
 840                         return -1;
 841                 }
 842                 if (nb_txq > port->dev_info.max_tx_queues) {
 843                         printf("Fail: nb_txq(%d) is greater than "
 844                                 "max_tx_queues(%d)\n", nb_txq,
 845                                 port->dev_info.max_tx_queues);
 846                         return -1;
 847                 }
 848                 if (numa_support) {
 849                         if (port_numa[pid] != NUMA_NO_CONFIG)
 850                                 port->socket_id = port_numa[pid];
 851                         else {
 852                                 port->socket_id = rte_eth_dev_socket_id(pid);
 853
 854                                 /* if socket_id is invalid, set to 0 */
 855                                 if (check_socket_id(port->socket_id) < 0)
 856                                         port->socket_id = 0;
 857                         }
 858                 }
 859                 else {
 860                         if (socket_num == UMA_NO_CONFIG)
 861                                 port->socket_id = 0;
 862                         else
 863                                 port->socket_id = socket_num;
 864                 }
 865         }
 866
 867         q = RTE_MAX(nb_rxq, nb_txq);
 868         if (q == 0) {
 869                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
 870                 return -1;
 871         }
 872         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
 873         if (nb_fwd_streams_new == nb_fwd_streams)
 874                 return 0;
 875         /* clear the old */
 876         if (fwd_streams != NULL) {
 877                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 878                         if (fwd_streams[sm_id] == NULL)
 879                                 continue;
 880                         rte_free(fwd_streams[sm_id]);
 881                         fwd_streams[sm_id] = NULL;
 882                 }
 883                 rte_free(fwd_streams);
 884                 fwd_streams = NULL;
 885         }
 886
 887         /* init new */
 888         nb_fwd_streams = nb_fwd_streams_new;
 889         if (nb_fwd_streams) {
 890                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
 891                         sizeof(struct fwd_stream *) * nb_fwd_streams,
 892                         RTE_CACHE_LINE_SIZE);
 893                 if (fwd_streams == NULL)
 894                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
 895                                  " (struct fwd_stream *)) failed\n",
 896                                  nb_fwd_streams);
 897
 898                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 899                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
 900                                 " struct fwd_stream", sizeof(struct fwd_stream),
 901                                 RTE_CACHE_LINE_SIZE);
 902                         if (fwd_streams[sm_id] == NULL)
 903                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
 904                                          "(struct fwd_stream) failed\n");
 905                 }
 906         }
 907
 908         return 0;
 909 }
 910
 911 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 912 static void
 913 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
 914 {
 915         unsigned int total_burst;
 916         unsigned int nb_burst;
 917         unsigned int burst_stats[3];
 918         uint16_t pktnb_stats[3];
 919         uint16_t nb_pkt;
 920         int burst_percent[3];
 921
 922         /*
 923          * First compute the total number of packet bursts and the
 924          * two highest numbers of bursts of the same number of packets.
 925          */
 926         total_burst = 0;
 927         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
 928         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
 929         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
 930                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
 931                 if (nb_burst == 0)
 932                         continue;
 933                 total_burst += nb_burst;
 934                 if (nb_burst > burst_stats[0]) {
 935                         burst_stats[1] = burst_stats[0];
 936                         pktnb_stats[1] = pktnb_stats[0];
 937                         burst_stats[0] = nb_burst;
 938                         pktnb_stats[0] = nb_pkt;
 939                 }
 940         }
 941         if (total_burst == 0)
 942                 return;
 943         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
 944         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
 945                burst_percent[0], (int) pktnb_stats[0]);
 946         if (burst_stats[0] == total_burst) {
 947                 printf("]\n");
 948                 return;
 949         }
 950         if (burst_stats[0] + burst_stats[1] == total_burst) {
 951                 printf(" + %d%% of %d pkts]\n",
 952                        100 - burst_percent[0], pktnb_stats[1]);
 953                 return;
 954         }
 955         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
 956         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
 957         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
 958                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
 959                 return;
 960         }
 961         printf(" + %d%% of %d pkts + %d%% of others]\n",
 962                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
 963 }
 964 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
 965
 966 static void
 967 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats)
 968 {
 969         struct rte_port *port;
 970         uint8_t i;
 971
 972         static const char *fwd_stats_border = "----------------------";
 973
 974         port = &ports[port_id];
 975         printf("\n  %s Forward statistics for port %-2d %s\n",
 976                fwd_stats_border, port_id, fwd_stats_border);
 977
 978         if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) {
 979                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
 980                        "%-"PRIu64"\n",
 981                        stats->ipackets, stats->imissed,
 982                        (uint64_t) (stats->ipackets + stats->imissed));
 983
 984                 if (cur_fwd_eng == &csum_fwd_engine)
 985                         printf("  Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n",
 986                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
 987                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
 988                         printf("  RX-error: %-"PRIu64"\n",  stats->ierrors);
 989                         printf("  RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf);
 990                 }
 991
 992                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
 993                        "%-"PRIu64"\n",
 994                        stats->opackets, port->tx_dropped,
 995                        (uint64_t) (stats->opackets + port->tx_dropped));
 996         }
 997         else {
 998                 printf("  RX-packets:             %14"PRIu64"    RX-dropped:%14"PRIu64"    RX-total:"
 999                        "%14"PRIu64"\n",
1000                        stats->ipackets, stats->imissed,
1001                        (uint64_t) (stats->ipackets + stats->imissed));
1002
1003                 if (cur_fwd_eng == &csum_fwd_engine)
1004                         printf("  Bad-ipcsum:%14"PRIu64"    Bad-l4csum:%14"PRIu64"\n",
1005                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
1006                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
1007                         printf("  RX-error:%"PRIu64"\n", stats->ierrors);
1008                         printf("  RX-nombufs:             %14"PRIu64"\n",
1009                                stats->rx_nombuf);
1010                 }
1011
1012                 printf("  TX-packets:             %14"PRIu64"    TX-dropped:%14"PRIu64"    TX-total:"
1013                        "%14"PRIu64"\n",
1014                        stats->opackets, port->tx_dropped,
1015                        (uint64_t) (stats->opackets + port->tx_dropped));
1016         }
1017
1018 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1019         if (port->rx_stream)
1020                 pkt_burst_stats_display("RX",
1021                         &port->rx_stream->rx_burst_stats);
1022         if (port->tx_stream)
1023                 pkt_burst_stats_display("TX",
1024                         &port->tx_stream->tx_burst_stats);
1025 #endif
1026
1027         if (port->rx_queue_stats_mapping_enabled) {
1028                 printf("\n");
1029                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
1030                         printf("  Stats reg %2d RX-packets:%14"PRIu64
1031                                "     RX-errors:%14"PRIu64
1032                                "    RX-bytes:%14"PRIu64"\n",
1033                                i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]);
1034                 }
1035                 printf("\n");
1036         }
1037         if (port->tx_queue_stats_mapping_enabled) {
1038                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
1039                         printf("  Stats reg %2d TX-packets:%14"PRIu64
1040                                "                                 TX-bytes:%14"PRIu64"\n",
1041                                i, stats->q_opackets[i], stats->q_obytes[i]);
1042                 }
1043         }
1044
1045         printf("  %s--------------------------------%s\n",
1046                fwd_stats_border, fwd_stats_border);
1047 }
1048
1049 static void
1050 fwd_stream_stats_display(streamid_t stream_id)
1051 {
1052         struct fwd_stream *fs;
1053         static const char *fwd_top_stats_border = "-------";
1054
1055         fs = fwd_streams[stream_id];
1056         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1057             (fs->fwd_dropped == 0))
1058                 return;
1059         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1060                "TX Port=%2d/Queue=%2d %s\n",
1061                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1062                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1063         printf("  RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u",
1064                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1065
1066         /* if checksum mode */
1067         if (cur_fwd_eng == &csum_fwd_engine) {
1068                printf("  RX- bad IP checksum: %-14u  Rx- bad L4 checksum: "
1069                         "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum);
1070         }
1071
1072 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1073         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1074         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1075 #endif
1076 }
1077
1078 static void
1079 flush_fwd_rx_queues(void)
1080 {
1081         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1082         portid_t  rxp;
1083         portid_t port_id;
1084         queueid_t rxq;
1085         uint16_t  nb_rx;
1086         uint16_t  i;
1087         uint8_t   j;
1088         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1089         uint64_t timer_period;
1090
1091         /* convert to number of cycles */
1092         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1093
1094         for (j = 0; j < 2; j++) {
1095                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1096                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1097                                 port_id = fwd_ports_ids[rxp];
1098                                 /**
1099                                 * testpmd can stuck in the below do while loop
1100                                 * if rte_eth_rx_burst() always returns nonzero
1101                                 * packets. So timer is added to exit this loop
1102                                 * after 1sec timer expiry.
1103                                 */
1104                                 prev_tsc = rte_rdtsc();
1105                                 do {
1106                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1107                                                 pkts_burst, MAX_PKT_BURST);
1108                                         for (i = 0; i < nb_rx; i++)
1109                                                 rte_pktmbuf_free(pkts_burst[i]);
1110
1111                                         cur_tsc = rte_rdtsc();
1112                                         diff_tsc = cur_tsc - prev_tsc;
1113                                         timer_tsc += diff_tsc;
1114                                 } while ((nb_rx > 0) &&
1115                                         (timer_tsc < timer_period));
1116                                 timer_tsc = 0;
1117                         }
1118                 }
1119                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1120         }
1121 }
1122
1123 static void
1124 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1125 {
1126         struct fwd_stream **fsm;
1127         streamid_t nb_fs;
1128         streamid_t sm_id;
1129 #ifdef RTE_LIBRTE_BITRATE
1130         uint64_t tics_per_1sec;
1131         uint64_t tics_datum;
1132         uint64_t tics_current;
1133         uint16_t idx_port;
1134
1135         tics_datum = rte_rdtsc();
1136         tics_per_1sec = rte_get_timer_hz();
1137 #endif
1138         fsm = &fwd_streams[fc->stream_idx];
1139         nb_fs = fc->stream_nb;
1140         do {
1141                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1142                         (*pkt_fwd)(fsm[sm_id]);
1143 #ifdef RTE_LIBRTE_BITRATE
1144                 if (bitrate_enabled != 0 &&
1145                                 bitrate_lcore_id == rte_lcore_id()) {
1146                         tics_current = rte_rdtsc();
1147                         if (tics_current - tics_datum >= tics_per_1sec) {
1148                                 /* Periodic bitrate calculation */
1149                                 RTE_ETH_FOREACH_DEV(idx_port)
1150                                         rte_stats_bitrate_calc(bitrate_data,
1151                                                 idx_port);
1152                                 tics_datum = tics_current;
1153                         }
1154                 }
1155 #endif
1156 #ifdef RTE_LIBRTE_LATENCY_STATS
1157                 if (latencystats_enabled != 0 &&
1158                                 latencystats_lcore_id == rte_lcore_id())
1159                         rte_latencystats_update();
1160 #endif
1161
1162         } while (! fc->stopped);
1163 }
1164
1165 static int
1166 start_pkt_forward_on_core(void *fwd_arg)
1167 {
1168         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1169                              cur_fwd_config.fwd_eng->packet_fwd);
1170         return 0;
1171 }
1172
1173 /*
1174  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1175  * Used to start communication flows in network loopback test configurations.
1176  */
1177 static int
1178 run_one_txonly_burst_on_core(void *fwd_arg)
1179 {
1180         struct fwd_lcore *fwd_lc;
1181         struct fwd_lcore tmp_lcore;
1182
1183         fwd_lc = (struct fwd_lcore *) fwd_arg;
1184         tmp_lcore = *fwd_lc;
1185         tmp_lcore.stopped = 1;
1186         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1187         return 0;
1188 }
1189
1190 /*
1191  * Launch packet forwarding:
1192  *     - Setup per-port forwarding context.
1193  *     - launch logical cores with their forwarding configuration.
1194  */
1195 static void
1196 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1197 {
1198         port_fwd_begin_t port_fwd_begin;
1199         unsigned int i;
1200         unsigned int lc_id;
1201         int diag;
1202
1203         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1204         if (port_fwd_begin != NULL) {
1205                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1206                         (*port_fwd_begin)(fwd_ports_ids[i]);
1207         }
1208         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1209                 lc_id = fwd_lcores_cpuids[i];
1210                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1211                         fwd_lcores[i]->stopped = 0;
1212                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1213                                                      fwd_lcores[i], lc_id);
1214                         if (diag != 0)
1215                                 printf("launch lcore %u failed - diag=%d\n",
1216                                        lc_id, diag);
1217                 }
1218         }
1219 }
1220
1221 /*
1222  * Launch packet forwarding configuration.
1223  */
1224 void
1225 start_packet_forwarding(int with_tx_first)
1226 {
1227         port_fwd_begin_t port_fwd_begin;
1228         port_fwd_end_t  port_fwd_end;
1229         struct rte_port *port;
1230         unsigned int i;
1231         portid_t   pt_id;
1232         streamid_t sm_id;
1233
1234         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1235                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1236
1237         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1238                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1239
1240         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1241                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1242                 (!nb_rxq || !nb_txq))
1243                 rte_exit(EXIT_FAILURE,
1244                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1245                         cur_fwd_eng->fwd_mode_name);
1246
1247         if (all_ports_started() == 0) {
1248                 printf("Not all ports were started\n");
1249                 return;
1250         }
1251         if (test_done == 0) {
1252                 printf("Packet forwarding already started\n");
1253                 return;
1254         }
1255
1256
1257         if(dcb_test) {
1258                 for (i = 0; i < nb_fwd_ports; i++) {
1259                         pt_id = fwd_ports_ids[i];
1260                         port = &ports[pt_id];
1261                         if (!port->dcb_flag) {
1262                                 printf("In DCB mode, all forwarding ports must "
1263                                        "be configured in this mode.\n");
1264                                 return;
1265                         }
1266                 }
1267                 if (nb_fwd_lcores == 1) {
1268                         printf("In DCB mode,the nb forwarding cores "
1269                                "should be larger than 1.\n");
1270                         return;
1271                 }
1272         }
1273         test_done = 0;
1274
1275         fwd_config_setup();
1276
1277         if(!no_flush_rx)
1278                 flush_fwd_rx_queues();
1279
1280         pkt_fwd_config_display(&cur_fwd_config);
1281         rxtx_config_display();
1282
1283         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1284                 pt_id = fwd_ports_ids[i];
1285                 port = &ports[pt_id];
1286                 rte_eth_stats_get(pt_id, &port->stats);
1287                 port->tx_dropped = 0;
1288
1289                 map_port_queue_stats_mapping_registers(pt_id, port);
1290         }
1291         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1292                 fwd_streams[sm_id]->rx_packets = 0;
1293                 fwd_streams[sm_id]->tx_packets = 0;
1294                 fwd_streams[sm_id]->fwd_dropped = 0;
1295                 fwd_streams[sm_id]->rx_bad_ip_csum = 0;
1296                 fwd_streams[sm_id]->rx_bad_l4_csum = 0;
1297
1298 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1299                 memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
1300                        sizeof(fwd_streams[sm_id]->rx_burst_stats));
1301                 memset(&fwd_streams[sm_id]->tx_burst_stats, 0,
1302                        sizeof(fwd_streams[sm_id]->tx_burst_stats));
1303 #endif
1304 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1305                 fwd_streams[sm_id]->core_cycles = 0;
1306 #endif
1307         }
1308         if (with_tx_first) {
1309                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1310                 if (port_fwd_begin != NULL) {
1311                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1312                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1313                 }
1314                 while (with_tx_first--) {
1315                         launch_packet_forwarding(
1316                                         run_one_txonly_burst_on_core);
1317                         rte_eal_mp_wait_lcore();
1318                 }
1319                 port_fwd_end = tx_only_engine.port_fwd_end;
1320                 if (port_fwd_end != NULL) {
1321                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1322                                 (*port_fwd_end)(fwd_ports_ids[i]);
1323                 }
1324         }
1325         launch_packet_forwarding(start_pkt_forward_on_core);
1326 }
1327
1328 void
1329 stop_packet_forwarding(void)
1330 {
1331         struct rte_eth_stats stats;
1332         struct rte_port *port;
1333         port_fwd_end_t  port_fwd_end;
1334         int i;
1335         portid_t   pt_id;
1336         streamid_t sm_id;
1337         lcoreid_t  lc_id;
1338         uint64_t total_recv;
1339         uint64_t total_xmit;
1340         uint64_t total_rx_dropped;
1341         uint64_t total_tx_dropped;
1342         uint64_t total_rx_nombuf;
1343         uint64_t tx_dropped;
1344         uint64_t rx_bad_ip_csum;
1345         uint64_t rx_bad_l4_csum;
1346 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1347         uint64_t fwd_cycles;
1348 #endif
1349
1350         static const char *acc_stats_border = "+++++++++++++++";
1351
1352         if (test_done) {
1353                 printf("Packet forwarding not started\n");
1354                 return;
1355         }
1356         printf("Telling cores to stop...");
1357         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1358                 fwd_lcores[lc_id]->stopped = 1;
1359         printf("\nWaiting for lcores to finish...\n");
1360         rte_eal_mp_wait_lcore();
1361         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1362         if (port_fwd_end != NULL) {
1363                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1364                         pt_id = fwd_ports_ids[i];
1365                         (*port_fwd_end)(pt_id);
1366                 }
1367         }
1368 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1369         fwd_cycles = 0;
1370 #endif
1371         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1372                 if (cur_fwd_config.nb_fwd_streams >
1373                     cur_fwd_config.nb_fwd_ports) {
1374                         fwd_stream_stats_display(sm_id);
1375                         ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL;
1376                         ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL;
1377                 } else {
1378                         ports[fwd_streams[sm_id]->tx_port].tx_stream =
1379                                 fwd_streams[sm_id];
1380                         ports[fwd_streams[sm_id]->rx_port].rx_stream =
1381                                 fwd_streams[sm_id];
1382                 }
1383                 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped;
1384                 tx_dropped = (uint64_t) (tx_dropped +
1385                                          fwd_streams[sm_id]->fwd_dropped);
1386                 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped;
1387
1388                 rx_bad_ip_csum =
1389                         ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum;
1390                 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum +
1391                                          fwd_streams[sm_id]->rx_bad_ip_csum);
1392                 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum =
1393                                                         rx_bad_ip_csum;
1394
1395                 rx_bad_l4_csum =
1396                         ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum;
1397                 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum +
1398                                          fwd_streams[sm_id]->rx_bad_l4_csum);
1399                 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum =
1400                                                         rx_bad_l4_csum;
1401
1402 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1403                 fwd_cycles = (uint64_t) (fwd_cycles +
1404                                          fwd_streams[sm_id]->core_cycles);
1405 #endif
1406         }
1407         total_recv = 0;
1408         total_xmit = 0;
1409         total_rx_dropped = 0;
1410         total_tx_dropped = 0;
1411         total_rx_nombuf  = 0;
1412         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1413                 pt_id = fwd_ports_ids[i];
1414
1415                 port = &ports[pt_id];
1416                 rte_eth_stats_get(pt_id, &stats);
1417                 stats.ipackets -= port->stats.ipackets;
1418                 port->stats.ipackets = 0;
1419                 stats.opackets -= port->stats.opackets;
1420                 port->stats.opackets = 0;
1421                 stats.ibytes   -= port->stats.ibytes;
1422                 port->stats.ibytes = 0;
1423                 stats.obytes   -= port->stats.obytes;
1424                 port->stats.obytes = 0;
1425                 stats.imissed  -= port->stats.imissed;
1426                 port->stats.imissed = 0;
1427                 stats.oerrors  -= port->stats.oerrors;
1428                 port->stats.oerrors = 0;
1429                 stats.rx_nombuf -= port->stats.rx_nombuf;
1430                 port->stats.rx_nombuf = 0;
1431
1432                 total_recv += stats.ipackets;
1433                 total_xmit += stats.opackets;
1434                 total_rx_dropped += stats.imissed;
1435                 total_tx_dropped += port->tx_dropped;
1436                 total_rx_nombuf  += stats.rx_nombuf;
1437
1438                 fwd_port_stats_display(pt_id, &stats);
1439         }
1440
1441         printf("\n  %s Accumulated forward statistics for all ports"
1442                "%s\n",
1443                acc_stats_border, acc_stats_border);
1444         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1445                "%-"PRIu64"\n"
1446                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1447                "%-"PRIu64"\n",
1448                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1449                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1450         if (total_rx_nombuf > 0)
1451                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1452         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1453                "%s\n",
1454                acc_stats_border, acc_stats_border);
1455 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1456         if (total_recv > 0)
1457                 printf("\n  CPU cycles/packet=%u (total cycles="
1458                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1459                        (unsigned int)(fwd_cycles / total_recv),
1460                        fwd_cycles, total_recv);
1461 #endif
1462         printf("\nDone.\n");
1463         test_done = 1;
1464 }
1465
1466 void
1467 dev_set_link_up(portid_t pid)
1468 {
1469         if (rte_eth_dev_set_link_up(pid) < 0)
1470                 printf("\nSet link up fail.\n");
1471 }
1472
1473 void
1474 dev_set_link_down(portid_t pid)
1475 {
1476         if (rte_eth_dev_set_link_down(pid) < 0)
1477                 printf("\nSet link down fail.\n");
1478 }
1479
1480 static int
1481 all_ports_started(void)
1482 {
1483         portid_t pi;
1484         struct rte_port *port;
1485
1486         RTE_ETH_FOREACH_DEV(pi) {
1487                 port = &ports[pi];
1488                 /* Check if there is a port which is not started */
1489                 if ((port->port_status != RTE_PORT_STARTED) &&
1490                         (port->slave_flag == 0))
1491                         return 0;
1492         }
1493
1494         /* No port is not started */
1495         return 1;
1496 }
1497
1498 int
1499 port_is_stopped(portid_t port_id)
1500 {
1501         struct rte_port *port = &ports[port_id];
1502
1503         if ((port->port_status != RTE_PORT_STOPPED) &&
1504             (port->slave_flag == 0))
1505                 return 0;
1506         return 1;
1507 }
1508
1509 int
1510 all_ports_stopped(void)
1511 {
1512         portid_t pi;
1513
1514         RTE_ETH_FOREACH_DEV(pi) {
1515                 if (!port_is_stopped(pi))
1516                         return 0;
1517         }
1518
1519         return 1;
1520 }
1521
1522 int
1523 port_is_started(portid_t port_id)
1524 {
1525         if (port_id_is_invalid(port_id, ENABLED_WARN))
1526                 return 0;
1527
1528         if (ports[port_id].port_status != RTE_PORT_STARTED)
1529                 return 0;
1530
1531         return 1;
1532 }
1533
1534 static int
1535 port_is_closed(portid_t port_id)
1536 {
1537         if (port_id_is_invalid(port_id, ENABLED_WARN))
1538                 return 0;
1539
1540         if (ports[port_id].port_status != RTE_PORT_CLOSED)
1541                 return 0;
1542
1543         return 1;
1544 }
1545
1546 int
1547 start_port(portid_t pid)
1548 {
1549         int diag, need_check_link_status = -1;
1550         portid_t pi;
1551         queueid_t qi;
1552         struct rte_port *port;
1553         struct ether_addr mac_addr;
1554         enum rte_eth_event_type event_type;
1555
1556         if (port_id_is_invalid(pid, ENABLED_WARN))
1557                 return 0;
1558
1559         if(dcb_config)
1560                 dcb_test = 1;
1561         RTE_ETH_FOREACH_DEV(pi) {
1562                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1563                         continue;
1564
1565                 need_check_link_status = 0;
1566                 port = &ports[pi];
1567                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1568                                                  RTE_PORT_HANDLING) == 0) {
1569                         printf("Port %d is now not stopped\n", pi);
1570                         continue;
1571                 }
1572
1573                 if (port->need_reconfig > 0) {
1574                         port->need_reconfig = 0;
1575
1576                         if (flow_isolate_all) {
1577                                 int ret = port_flow_isolate(pi, 1);
1578                                 if (ret) {
1579                                         printf("Failed to apply isolated"
1580                                                " mode on port %d\n", pi);
1581                                         return -1;
1582                                 }
1583                         }
1584
1585                         printf("Configuring Port %d (socket %u)\n", pi,
1586                                         port->socket_id);
1587                         /* configure port */
1588                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1589                                                 &(port->dev_conf));
1590                         if (diag != 0) {
1591                                 if (rte_atomic16_cmpset(&(port->port_status),
1592                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1593                                         printf("Port %d can not be set back "
1594                                                         "to stopped\n", pi);
1595                                 printf("Fail to configure port %d\n", pi);
1596                                 /* try to reconfigure port next time */
1597                                 port->need_reconfig = 1;
1598                                 return -1;
1599                         }
1600                 }
1601                 if (port->need_reconfig_queues > 0) {
1602                         port->need_reconfig_queues = 0;
1603                         /* setup tx queues */
1604                         for (qi = 0; qi < nb_txq; qi++) {
1605                                 port->tx_conf[qi].txq_flags =
1606                                         ETH_TXQ_FLAGS_IGNORE;
1607                                 /* Apply Tx offloads configuration */
1608                                 port->tx_conf[qi].offloads =
1609                                         port->dev_conf.txmode.offloads;
1610                                 if ((numa_support) &&
1611                                         (txring_numa[pi] != NUMA_NO_CONFIG))
1612                                         diag = rte_eth_tx_queue_setup(pi, qi,
1613                                                 port->nb_tx_desc[qi],
1614                                                 txring_numa[pi],
1615                                                 &(port->tx_conf[qi]));
1616                                 else
1617                                         diag = rte_eth_tx_queue_setup(pi, qi,
1618                                                 port->nb_tx_desc[qi],
1619                                                 port->socket_id,
1620                                                 &(port->tx_conf[qi]));
1621
1622                                 if (diag == 0)
1623                                         continue;
1624
1625                                 /* Fail to setup tx queue, return */
1626                                 if (rte_atomic16_cmpset(&(port->port_status),
1627                                                         RTE_PORT_HANDLING,
1628                                                         RTE_PORT_STOPPED) == 0)
1629                                         printf("Port %d can not be set back "
1630                                                         "to stopped\n", pi);
1631                                 printf("Fail to configure port %d tx queues\n",
1632                                        pi);
1633                                 /* try to reconfigure queues next time */
1634                                 port->need_reconfig_queues = 1;
1635                                 return -1;
1636                         }
1637                         for (qi = 0; qi < nb_rxq; qi++) {
1638                                 /* Apply Rx offloads configuration */
1639                                 port->rx_conf[qi].offloads =
1640                                         port->dev_conf.rxmode.offloads;
1641                                 /* setup rx queues */
1642                                 if ((numa_support) &&
1643                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
1644                                         struct rte_mempool * mp =
1645                                                 mbuf_pool_find(rxring_numa[pi]);
1646                                         if (mp == NULL) {
1647                                                 printf("Failed to setup RX queue:"
1648                                                         "No mempool allocation"
1649                                                         " on the socket %d\n",
1650                                                         rxring_numa[pi]);
1651                                                 return -1;
1652                                         }
1653
1654                                         diag = rte_eth_rx_queue_setup(pi, qi,
1655                                              port->nb_rx_desc[pi],
1656                                              rxring_numa[pi],
1657                                              &(port->rx_conf[qi]),
1658                                              mp);
1659                                 } else {
1660                                         struct rte_mempool *mp =
1661                                                 mbuf_pool_find(port->socket_id);
1662                                         if (mp == NULL) {
1663                                                 printf("Failed to setup RX queue:"
1664                                                         "No mempool allocation"
1665                                                         " on the socket %d\n",
1666                                                         port->socket_id);
1667                                                 return -1;
1668                                         }
1669                                         diag = rte_eth_rx_queue_setup(pi, qi,
1670                                              port->nb_rx_desc[pi],
1671                                              port->socket_id,
1672                                              &(port->rx_conf[qi]),
1673                                              mp);
1674                                 }
1675                                 if (diag == 0)
1676                                         continue;
1677
1678                                 /* Fail to setup rx queue, return */
1679                                 if (rte_atomic16_cmpset(&(port->port_status),
1680                                                         RTE_PORT_HANDLING,
1681                                                         RTE_PORT_STOPPED) == 0)
1682                                         printf("Port %d can not be set back "
1683                                                         "to stopped\n", pi);
1684                                 printf("Fail to configure port %d rx queues\n",
1685                                        pi);
1686                                 /* try to reconfigure queues next time */
1687                                 port->need_reconfig_queues = 1;
1688                                 return -1;
1689                         }
1690                 }
1691
1692                 /* start port */
1693                 if (rte_eth_dev_start(pi) < 0) {
1694                         printf("Fail to start port %d\n", pi);
1695
1696                         /* Fail to setup rx queue, return */
1697                         if (rte_atomic16_cmpset(&(port->port_status),
1698                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1699                                 printf("Port %d can not be set back to "
1700                                                         "stopped\n", pi);
1701                         continue;
1702                 }
1703
1704                 if (rte_atomic16_cmpset(&(port->port_status),
1705                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
1706                         printf("Port %d can not be set into started\n", pi);
1707
1708                 rte_eth_macaddr_get(pi, &mac_addr);
1709                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
1710                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
1711                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
1712                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
1713
1714                 /* at least one port started, need checking link status */
1715                 need_check_link_status = 1;
1716         }
1717
1718         for (event_type = RTE_ETH_EVENT_UNKNOWN;
1719              event_type < RTE_ETH_EVENT_MAX;
1720              event_type++) {
1721                 diag = rte_eth_dev_callback_register(RTE_ETH_ALL,
1722                                                 event_type,
1723                                                 eth_event_callback,
1724                                                 NULL);
1725                 if (diag) {
1726                         printf("Failed to setup even callback for event %d\n",
1727                                 event_type);
1728                         return -1;
1729                 }
1730         }
1731
1732         if (need_check_link_status == 1 && !no_link_check)
1733                 check_all_ports_link_status(RTE_PORT_ALL);
1734         else if (need_check_link_status == 0)
1735                 printf("Please stop the ports first\n");
1736
1737         printf("Done\n");
1738         return 0;
1739 }
1740
1741 void
1742 stop_port(portid_t pid)
1743 {
1744         portid_t pi;
1745         struct rte_port *port;
1746         int need_check_link_status = 0;
1747
1748         if (dcb_test) {
1749                 dcb_test = 0;
1750                 dcb_config = 0;
1751         }
1752
1753         if (port_id_is_invalid(pid, ENABLED_WARN))
1754                 return;
1755
1756         printf("Stopping ports...\n");
1757
1758         RTE_ETH_FOREACH_DEV(pi) {
1759                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1760                         continue;
1761
1762                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1763                         printf("Please remove port %d from forwarding configuration.\n", pi);
1764                         continue;
1765                 }
1766
1767                 if (port_is_bonding_slave(pi)) {
1768                         printf("Please remove port %d from bonded device.\n", pi);
1769                         continue;
1770                 }
1771
1772                 port = &ports[pi];
1773                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
1774                                                 RTE_PORT_HANDLING) == 0)
1775                         continue;
1776
1777                 rte_eth_dev_stop(pi);
1778
1779                 if (rte_atomic16_cmpset(&(port->port_status),
1780                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1781                         printf("Port %d can not be set into stopped\n", pi);
1782                 need_check_link_status = 1;
1783         }
1784         if (need_check_link_status && !no_link_check)
1785                 check_all_ports_link_status(RTE_PORT_ALL);
1786
1787         printf("Done\n");
1788 }
1789
1790 void
1791 close_port(portid_t pid)
1792 {
1793         portid_t pi;
1794         struct rte_port *port;
1795
1796         if (port_id_is_invalid(pid, ENABLED_WARN))
1797                 return;
1798
1799         printf("Closing ports...\n");
1800
1801         RTE_ETH_FOREACH_DEV(pi) {
1802                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1803                         continue;
1804
1805                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1806                         printf("Please remove port %d from forwarding configuration.\n", pi);
1807                         continue;
1808                 }
1809
1810                 if (port_is_bonding_slave(pi)) {
1811                         printf("Please remove port %d from bonded device.\n", pi);
1812                         continue;
1813                 }
1814
1815                 port = &ports[pi];
1816                 if (rte_atomic16_cmpset(&(port->port_status),
1817                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
1818                         printf("Port %d is already closed\n", pi);
1819                         continue;
1820                 }
1821
1822                 if (rte_atomic16_cmpset(&(port->port_status),
1823                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
1824                         printf("Port %d is now not stopped\n", pi);
1825                         continue;
1826                 }
1827
1828                 if (port->flow_list)
1829                         port_flow_flush(pi);
1830                 rte_eth_dev_close(pi);
1831
1832                 if (rte_atomic16_cmpset(&(port->port_status),
1833                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
1834                         printf("Port %d cannot be set to closed\n", pi);
1835         }
1836
1837         printf("Done\n");
1838 }
1839
1840 void
1841 reset_port(portid_t pid)
1842 {
1843         int diag;
1844         portid_t pi;
1845         struct rte_port *port;
1846
1847         if (port_id_is_invalid(pid, ENABLED_WARN))
1848                 return;
1849
1850         printf("Resetting ports...\n");
1851
1852         RTE_ETH_FOREACH_DEV(pi) {
1853                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1854                         continue;
1855
1856                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1857                         printf("Please remove port %d from forwarding "
1858                                "configuration.\n", pi);
1859                         continue;
1860                 }
1861
1862                 if (port_is_bonding_slave(pi)) {
1863                         printf("Please remove port %d from bonded device.\n",
1864                                pi);
1865                         continue;
1866                 }
1867
1868                 diag = rte_eth_dev_reset(pi);
1869                 if (diag == 0) {
1870                         port = &ports[pi];
1871                         port->need_reconfig = 1;
1872                         port->need_reconfig_queues = 1;
1873                 } else {
1874                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
1875                 }
1876         }
1877
1878         printf("Done\n");
1879 }
1880
1881 static int
1882 eth_dev_event_callback_register(void)
1883 {
1884         int ret;
1885
1886         /* register the device event callback */
1887         ret = rte_dev_event_callback_register(NULL,
1888                 eth_dev_event_callback, NULL);
1889         if (ret) {
1890                 printf("Failed to register device event callback\n");
1891                 return -1;
1892         }
1893
1894         return 0;
1895 }
1896
1897
1898 static int
1899 eth_dev_event_callback_unregister(void)
1900 {
1901         int ret;
1902
1903         /* unregister the device event callback */
1904         ret = rte_dev_event_callback_unregister(NULL,
1905                 eth_dev_event_callback, NULL);
1906         if (ret < 0) {
1907                 printf("Failed to unregister device event callback\n");
1908                 return -1;
1909         }
1910
1911         return 0;
1912 }
1913
1914 void
1915 attach_port(char *identifier)
1916 {
1917         portid_t pi = 0;
1918         unsigned int socket_id;
1919
1920         printf("Attaching a new port...\n");
1921
1922         if (identifier == NULL) {
1923                 printf("Invalid parameters are specified\n");
1924                 return;
1925         }
1926
1927         if (rte_eth_dev_attach(identifier, &pi))
1928                 return;
1929
1930         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
1931         /* if socket_id is invalid, set to 0 */
1932         if (check_socket_id(socket_id) < 0)
1933                 socket_id = 0;
1934         reconfig(pi, socket_id);
1935         rte_eth_promiscuous_enable(pi);
1936
1937         nb_ports = rte_eth_dev_count_avail();
1938
1939         ports[pi].port_status = RTE_PORT_STOPPED;
1940
1941         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
1942         printf("Done\n");
1943 }
1944
1945 void
1946 detach_port(portid_t port_id)
1947 {
1948         char name[RTE_ETH_NAME_MAX_LEN];
1949
1950         printf("Detaching a port...\n");
1951
1952         if (!port_is_closed(port_id)) {
1953                 printf("Please close port first\n");
1954                 return;
1955         }
1956
1957         if (ports[port_id].flow_list)
1958                 port_flow_flush(port_id);
1959
1960         if (rte_eth_dev_detach(port_id, name)) {
1961                 TESTPMD_LOG(ERR, "Failed to detach port '%s'\n", name);
1962                 return;
1963         }
1964
1965         nb_ports = rte_eth_dev_count_avail();
1966
1967         printf("Port '%s' is detached. Now total ports is %d\n",
1968                         name, nb_ports);
1969         printf("Done\n");
1970         return;
1971 }
1972
1973 void
1974 pmd_test_exit(void)
1975 {
1976         portid_t pt_id;
1977         int ret;
1978
1979         if (test_done == 0)
1980                 stop_packet_forwarding();
1981
1982         if (ports != NULL) {
1983                 no_link_check = 1;
1984                 RTE_ETH_FOREACH_DEV(pt_id) {
1985                         printf("\nShutting down port %d...\n", pt_id);
1986                         fflush(stdout);
1987                         stop_port(pt_id);
1988                         close_port(pt_id);
1989                 }
1990         }
1991
1992         if (hot_plug) {
1993                 ret = rte_dev_event_monitor_stop();
1994                 if (ret)
1995                         RTE_LOG(ERR, EAL,
1996                                 "fail to stop device event monitor.");
1997
1998                 ret = eth_dev_event_callback_unregister();
1999                 if (ret)
2000                         RTE_LOG(ERR, EAL,
2001                                 "fail to unregister all event callbacks.");
2002         }
2003
2004         printf("\nBye...\n");
2005 }
2006
2007 typedef void (*cmd_func_t)(void);
2008 struct pmd_test_command {
2009         const char *cmd_name;
2010         cmd_func_t cmd_func;
2011 };
2012
2013 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2014
2015 /* Check the link status of all ports in up to 9s, and print them finally */
2016 static void
2017 check_all_ports_link_status(uint32_t port_mask)
2018 {
2019 #define CHECK_INTERVAL 100 /* 100ms */
2020 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2021         portid_t portid;
2022         uint8_t count, all_ports_up, print_flag = 0;
2023         struct rte_eth_link link;
2024
2025         printf("Checking link statuses...\n");
2026         fflush(stdout);
2027         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2028                 all_ports_up = 1;
2029                 RTE_ETH_FOREACH_DEV(portid) {
2030                         if ((port_mask & (1 << portid)) == 0)
2031                                 continue;
2032                         memset(&link, 0, sizeof(link));
2033                         rte_eth_link_get_nowait(portid, &link);
2034                         /* print link status if flag set */
2035                         if (print_flag == 1) {
2036                                 if (link.link_status)
2037                                         printf(
2038                                         "Port%d Link Up. speed %u Mbps- %s\n",
2039                                         portid, link.link_speed,
2040                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2041                                         ("full-duplex") : ("half-duplex\n"));
2042                                 else
2043                                         printf("Port %d Link Down\n", portid);
2044                                 continue;
2045                         }
2046                         /* clear all_ports_up flag if any link down */
2047                         if (link.link_status == ETH_LINK_DOWN) {
2048                                 all_ports_up = 0;
2049                                 break;
2050                         }
2051                 }
2052                 /* after finally printing all link status, get out */
2053                 if (print_flag == 1)
2054                         break;
2055
2056                 if (all_ports_up == 0) {
2057                         fflush(stdout);
2058                         rte_delay_ms(CHECK_INTERVAL);
2059                 }
2060
2061                 /* set the print_flag if all ports up or timeout */
2062                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2063                         print_flag = 1;
2064                 }
2065
2066                 if (lsc_interrupt)
2067                         break;
2068         }
2069 }
2070
2071 static void
2072 rmv_event_callback(void *arg)
2073 {
2074         struct rte_eth_dev *dev;
2075         portid_t port_id = (intptr_t)arg;
2076
2077         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2078         dev = &rte_eth_devices[port_id];
2079
2080         stop_port(port_id);
2081         close_port(port_id);
2082         printf("removing device %s\n", dev->device->name);
2083         if (rte_eal_dev_detach(dev->device))
2084                 TESTPMD_LOG(ERR, "Failed to detach device %s\n",
2085                         dev->device->name);
2086 }
2087
2088 /* This function is used by the interrupt thread */
2089 static int
2090 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2091                   void *ret_param)
2092 {
2093         static const char * const event_desc[] = {
2094                 [RTE_ETH_EVENT_UNKNOWN] = "Unknown",
2095                 [RTE_ETH_EVENT_INTR_LSC] = "LSC",
2096                 [RTE_ETH_EVENT_QUEUE_STATE] = "Queue state",
2097                 [RTE_ETH_EVENT_INTR_RESET] = "Interrupt reset",
2098                 [RTE_ETH_EVENT_VF_MBOX] = "VF Mbox",
2099                 [RTE_ETH_EVENT_IPSEC] = "IPsec",
2100                 [RTE_ETH_EVENT_MACSEC] = "MACsec",
2101                 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
2102                 [RTE_ETH_EVENT_NEW] = "device probed",
2103                 [RTE_ETH_EVENT_DESTROY] = "device released",
2104                 [RTE_ETH_EVENT_MAX] = NULL,
2105         };
2106
2107         RTE_SET_USED(param);
2108         RTE_SET_USED(ret_param);
2109
2110         if (type >= RTE_ETH_EVENT_MAX) {
2111                 fprintf(stderr, "\nPort %" PRIu8 ": %s called upon invalid event %d\n",
2112                         port_id, __func__, type);
2113                 fflush(stderr);
2114         } else if (event_print_mask & (UINT32_C(1) << type)) {
2115                 printf("\nPort %" PRIu8 ": %s event\n", port_id,
2116                         event_desc[type]);
2117                 fflush(stdout);
2118         }
2119
2120         if (port_id_is_invalid(port_id, DISABLED_WARN))
2121                 return 0;
2122
2123         switch (type) {
2124         case RTE_ETH_EVENT_INTR_RMV:
2125                 if (rte_eal_alarm_set(100000,
2126                                 rmv_event_callback, (void *)(intptr_t)port_id))
2127                         fprintf(stderr, "Could not set up deferred device removal\n");
2128                 break;
2129         default:
2130                 break;
2131         }
2132         return 0;
2133 }
2134
2135 /* This function is used by the interrupt thread */
2136 static void
2137 eth_dev_event_callback(char *device_name, enum rte_dev_event_type type,
2138                              __rte_unused void *arg)
2139 {
2140         if (type >= RTE_DEV_EVENT_MAX) {
2141                 fprintf(stderr, "%s called upon invalid event %d\n",
2142                         __func__, type);
2143                 fflush(stderr);
2144         }
2145
2146         switch (type) {
2147         case RTE_DEV_EVENT_REMOVE:
2148                 RTE_LOG(ERR, EAL, "The device: %s has been removed!\n",
2149                         device_name);
2150                 /* TODO: After finish failure handle, begin to stop
2151                  * packet forward, stop port, close port, detach port.
2152                  */
2153                 break;
2154         case RTE_DEV_EVENT_ADD:
2155                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2156                         device_name);
2157                 /* TODO: After finish kernel driver binding,
2158                  * begin to attach port.
2159                  */
2160                 break;
2161         default:
2162                 break;
2163         }
2164 }
2165
2166 static int
2167 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2168 {
2169         uint16_t i;
2170         int diag;
2171         uint8_t mapping_found = 0;
2172
2173         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2174                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2175                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2176                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2177                                         tx_queue_stats_mappings[i].queue_id,
2178                                         tx_queue_stats_mappings[i].stats_counter_id);
2179                         if (diag != 0)
2180                                 return diag;
2181                         mapping_found = 1;
2182                 }
2183         }
2184         if (mapping_found)
2185                 port->tx_queue_stats_mapping_enabled = 1;
2186         return 0;
2187 }
2188
2189 static int
2190 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2191 {
2192         uint16_t i;
2193         int diag;
2194         uint8_t mapping_found = 0;
2195
2196         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2197                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2198                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2199                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2200                                         rx_queue_stats_mappings[i].queue_id,
2201                                         rx_queue_stats_mappings[i].stats_counter_id);
2202                         if (diag != 0)
2203                                 return diag;
2204                         mapping_found = 1;
2205                 }
2206         }
2207         if (mapping_found)
2208                 port->rx_queue_stats_mapping_enabled = 1;
2209         return 0;
2210 }
2211
2212 static void
2213 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2214 {
2215         int diag = 0;
2216
2217         diag = set_tx_queue_stats_mapping_registers(pi, port);
2218         if (diag != 0) {
2219                 if (diag == -ENOTSUP) {
2220                         port->tx_queue_stats_mapping_enabled = 0;
2221                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2222                 }
2223                 else
2224                         rte_exit(EXIT_FAILURE,
2225                                         "set_tx_queue_stats_mapping_registers "
2226                                         "failed for port id=%d diag=%d\n",
2227                                         pi, diag);
2228         }
2229
2230         diag = set_rx_queue_stats_mapping_registers(pi, port);
2231         if (diag != 0) {
2232                 if (diag == -ENOTSUP) {
2233                         port->rx_queue_stats_mapping_enabled = 0;
2234                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2235                 }
2236                 else
2237                         rte_exit(EXIT_FAILURE,
2238                                         "set_rx_queue_stats_mapping_registers "
2239                                         "failed for port id=%d diag=%d\n",
2240                                         pi, diag);
2241         }
2242 }
2243
2244 static void
2245 rxtx_port_config(struct rte_port *port)
2246 {
2247         uint16_t qid;
2248
2249         for (qid = 0; qid < nb_rxq; qid++) {
2250                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2251
2252                 /* Check if any Rx parameters have been passed */
2253                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2254                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2255
2256                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2257                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2258
2259                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2260                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2261
2262                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2263                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2264
2265                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2266                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2267
2268                 port->nb_rx_desc[qid] = nb_rxd;
2269         }
2270
2271         for (qid = 0; qid < nb_txq; qid++) {
2272                 port->tx_conf[qid] = port->dev_info.default_txconf;
2273
2274                 /* Check if any Tx parameters have been passed */
2275                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2276                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2277
2278                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2279                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2280
2281                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2282                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2283
2284                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2285                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2286
2287                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2288                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2289
2290                 port->nb_tx_desc[qid] = nb_txd;
2291         }
2292 }
2293
2294 void
2295 init_port_config(void)
2296 {
2297         portid_t pid;
2298         struct rte_port *port;
2299         struct rte_eth_dev_info dev_info;
2300
2301         RTE_ETH_FOREACH_DEV(pid) {
2302                 port = &ports[pid];
2303                 port->dev_conf.fdir_conf = fdir_conf;
2304                 if (nb_rxq > 1) {
2305                         rte_eth_dev_info_get(pid, &dev_info);
2306                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2307                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2308                                 rss_hf & dev_info.flow_type_rss_offloads;
2309                 } else {
2310                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2311                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2312                 }
2313
2314                 if (port->dcb_flag == 0) {
2315                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2316                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2317                         else
2318                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2319                 }
2320
2321                 rxtx_port_config(port);
2322
2323                 rte_eth_macaddr_get(pid, &port->eth_addr);
2324
2325                 map_port_queue_stats_mapping_registers(pid, port);
2326 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2327                 rte_pmd_ixgbe_bypass_init(pid);
2328 #endif
2329
2330                 if (lsc_interrupt &&
2331                     (rte_eth_devices[pid].data->dev_flags &
2332                      RTE_ETH_DEV_INTR_LSC))
2333                         port->dev_conf.intr_conf.lsc = 1;
2334                 if (rmv_interrupt &&
2335                     (rte_eth_devices[pid].data->dev_flags &
2336                      RTE_ETH_DEV_INTR_RMV))
2337                         port->dev_conf.intr_conf.rmv = 1;
2338
2339 #if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED
2340                 /* Detect softnic port */
2341                 if (!strcmp(port->dev_info.driver_name, "net_softnic")) {
2342                         port->softnic_enable = 1;
2343                         memset(&port->softport, 0, sizeof(struct softnic_port));
2344
2345                         if (!strcmp(cur_fwd_eng->fwd_mode_name, "tm"))
2346                                 port->softport.tm_flag = 1;
2347                 }
2348 #endif
2349         }
2350 }
2351
2352 void set_port_slave_flag(portid_t slave_pid)
2353 {
2354         struct rte_port *port;
2355
2356         port = &ports[slave_pid];
2357         port->slave_flag = 1;
2358 }
2359
2360 void clear_port_slave_flag(portid_t slave_pid)
2361 {
2362         struct rte_port *port;
2363
2364         port = &ports[slave_pid];
2365         port->slave_flag = 0;
2366 }
2367
2368 uint8_t port_is_bonding_slave(portid_t slave_pid)
2369 {
2370         struct rte_port *port;
2371
2372         port = &ports[slave_pid];
2373         if ((rte_eth_devices[slave_pid].data->dev_flags &
2374             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
2375                 return 1;
2376         return 0;
2377 }
2378
2379 const uint16_t vlan_tags[] = {
2380                 0,  1,  2,  3,  4,  5,  6,  7,
2381                 8,  9, 10, 11,  12, 13, 14, 15,
2382                 16, 17, 18, 19, 20, 21, 22, 23,
2383                 24, 25, 26, 27, 28, 29, 30, 31
2384 };
2385
2386 static  int
2387 get_eth_dcb_conf(struct rte_eth_conf *eth_conf,
2388                  enum dcb_mode_enable dcb_mode,
2389                  enum rte_eth_nb_tcs num_tcs,
2390                  uint8_t pfc_en)
2391 {
2392         uint8_t i;
2393
2394         /*
2395          * Builds up the correct configuration for dcb+vt based on the vlan tags array
2396          * given above, and the number of traffic classes available for use.
2397          */
2398         if (dcb_mode == DCB_VT_ENABLED) {
2399                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
2400                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
2401                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2402                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
2403
2404                 /* VMDQ+DCB RX and TX configurations */
2405                 vmdq_rx_conf->enable_default_pool = 0;
2406                 vmdq_rx_conf->default_pool = 0;
2407                 vmdq_rx_conf->nb_queue_pools =
2408                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2409                 vmdq_tx_conf->nb_queue_pools =
2410                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2411
2412                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
2413                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
2414                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
2415                         vmdq_rx_conf->pool_map[i].pools =
2416                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
2417                 }
2418                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2419                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
2420                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
2421                 }
2422
2423                 /* set DCB mode of RX and TX of multiple queues */
2424                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
2425                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
2426         } else {
2427                 struct rte_eth_dcb_rx_conf *rx_conf =
2428                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
2429                 struct rte_eth_dcb_tx_conf *tx_conf =
2430                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
2431
2432                 rx_conf->nb_tcs = num_tcs;
2433                 tx_conf->nb_tcs = num_tcs;
2434
2435                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2436                         rx_conf->dcb_tc[i] = i % num_tcs;
2437                         tx_conf->dcb_tc[i] = i % num_tcs;
2438                 }
2439                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
2440                 eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf;
2441                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
2442         }
2443
2444         if (pfc_en)
2445                 eth_conf->dcb_capability_en =
2446                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
2447         else
2448                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
2449
2450         return 0;
2451 }
2452
2453 int
2454 init_port_dcb_config(portid_t pid,
2455                      enum dcb_mode_enable dcb_mode,
2456                      enum rte_eth_nb_tcs num_tcs,
2457                      uint8_t pfc_en)
2458 {
2459         struct rte_eth_conf port_conf;
2460         struct rte_port *rte_port;
2461         int retval;
2462         uint16_t i;
2463
2464         rte_port = &ports[pid];
2465
2466         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
2467         /* Enter DCB configuration status */
2468         dcb_config = 1;
2469
2470         port_conf.rxmode = rte_port->dev_conf.rxmode;
2471         port_conf.txmode = rte_port->dev_conf.txmode;
2472
2473         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
2474         retval = get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en);
2475         if (retval < 0)
2476                 return retval;
2477         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
2478
2479         /**
2480          * Write the configuration into the device.
2481          * Set the numbers of RX & TX queues to 0, so
2482          * the RX & TX queues will not be setup.
2483          */
2484         rte_eth_dev_configure(pid, 0, 0, &port_conf);
2485
2486         rte_eth_dev_info_get(pid, &rte_port->dev_info);
2487
2488         /* If dev_info.vmdq_pool_base is greater than 0,
2489          * the queue id of vmdq pools is started after pf queues.
2490          */
2491         if (dcb_mode == DCB_VT_ENABLED &&
2492             rte_port->dev_info.vmdq_pool_base > 0) {
2493                 printf("VMDQ_DCB multi-queue mode is nonsensical"
2494                         " for port %d.", pid);
2495                 return -1;
2496         }
2497
2498         /* Assume the ports in testpmd have the same dcb capability
2499          * and has the same number of rxq and txq in dcb mode
2500          */
2501         if (dcb_mode == DCB_VT_ENABLED) {
2502                 if (rte_port->dev_info.max_vfs > 0) {
2503                         nb_rxq = rte_port->dev_info.nb_rx_queues;
2504                         nb_txq = rte_port->dev_info.nb_tx_queues;
2505                 } else {
2506                         nb_rxq = rte_port->dev_info.max_rx_queues;
2507                         nb_txq = rte_port->dev_info.max_tx_queues;
2508                 }
2509         } else {
2510                 /*if vt is disabled, use all pf queues */
2511                 if (rte_port->dev_info.vmdq_pool_base == 0) {
2512                         nb_rxq = rte_port->dev_info.max_rx_queues;
2513                         nb_txq = rte_port->dev_info.max_tx_queues;
2514                 } else {
2515                         nb_rxq = (queueid_t)num_tcs;
2516                         nb_txq = (queueid_t)num_tcs;
2517
2518                 }
2519         }
2520         rx_free_thresh = 64;
2521
2522         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
2523
2524         rxtx_port_config(rte_port);
2525         /* VLAN filter */
2526         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
2527         for (i = 0; i < RTE_DIM(vlan_tags); i++)
2528                 rx_vft_set(pid, vlan_tags[i], 1);
2529
2530         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
2531         map_port_queue_stats_mapping_registers(pid, rte_port);
2532
2533         rte_port->dcb_flag = 1;
2534
2535         return 0;
2536 }
2537
2538 static void
2539 init_port(void)
2540 {
2541         /* Configuration of Ethernet ports. */
2542         ports = rte_zmalloc("testpmd: ports",
2543                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
2544                             RTE_CACHE_LINE_SIZE);
2545         if (ports == NULL) {
2546                 rte_exit(EXIT_FAILURE,
2547                                 "rte_zmalloc(%d struct rte_port) failed\n",
2548                                 RTE_MAX_ETHPORTS);
2549         }
2550 }
2551
2552 static void
2553 force_quit(void)
2554 {
2555         pmd_test_exit();
2556         prompt_exit();
2557 }
2558
2559 static void
2560 print_stats(void)
2561 {
2562         uint8_t i;
2563         const char clr[] = { 27, '[', '2', 'J', '\0' };
2564         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
2565
2566         /* Clear screen and move to top left */
2567         printf("%s%s", clr, top_left);
2568
2569         printf("\nPort statistics ====================================");
2570         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2571                 nic_stats_display(fwd_ports_ids[i]);
2572 }
2573
2574 static void
2575 signal_handler(int signum)
2576 {
2577         if (signum == SIGINT || signum == SIGTERM) {
2578                 printf("\nSignal %d received, preparing to exit...\n",
2579                                 signum);
2580 #ifdef RTE_LIBRTE_PDUMP
2581                 /* uninitialize packet capture framework */
2582                 rte_pdump_uninit();
2583 #endif
2584 #ifdef RTE_LIBRTE_LATENCY_STATS
2585                 rte_latencystats_uninit();
2586 #endif
2587                 force_quit();
2588                 /* Set flag to indicate the force termination. */
2589                 f_quit = 1;
2590                 /* exit with the expected status */
2591                 signal(signum, SIG_DFL);
2592                 kill(getpid(), signum);
2593         }
2594 }
2595
2596 int
2597 main(int argc, char** argv)
2598 {
2599         int diag;
2600         portid_t port_id;
2601         int ret;
2602
2603         signal(SIGINT, signal_handler);
2604         signal(SIGTERM, signal_handler);
2605
2606         diag = rte_eal_init(argc, argv);
2607         if (diag < 0)
2608                 rte_panic("Cannot init EAL\n");
2609
2610         testpmd_logtype = rte_log_register("testpmd");
2611         if (testpmd_logtype < 0)
2612                 rte_panic("Cannot register log type");
2613         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
2614
2615         /* Bitrate/latency stats disabled by default */
2616 #ifdef RTE_LIBRTE_BITRATE
2617         bitrate_enabled = 0;
2618 #endif
2619 #ifdef RTE_LIBRTE_LATENCY_STATS
2620         latencystats_enabled = 0;
2621 #endif
2622
2623         /* on FreeBSD, mlockall() is disabled by default */
2624 #ifdef RTE_EXEC_ENV_BSDAPP
2625         do_mlockall = 0;
2626 #else
2627         do_mlockall = 1;
2628 #endif
2629
2630         argc -= diag;
2631         argv += diag;
2632         if (argc > 1)
2633                 launch_args_parse(argc, argv);
2634
2635         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
2636                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
2637                         strerror(errno));
2638         }
2639
2640 #ifdef RTE_LIBRTE_PDUMP
2641         /* initialize packet capture framework */
2642         rte_pdump_init(NULL);
2643 #endif
2644
2645         nb_ports = (portid_t) rte_eth_dev_count_avail();
2646         if (nb_ports == 0)
2647                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
2648
2649         /* allocate port structures, and init them */
2650         init_port();
2651
2652         set_def_fwd_config();
2653         if (nb_lcores == 0)
2654                 rte_panic("Empty set of forwarding logical cores - check the "
2655                           "core mask supplied in the command parameters\n");
2656
2657         if (tx_first && interactive)
2658                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
2659                                 "interactive mode.\n");
2660
2661         if (tx_first && lsc_interrupt) {
2662                 printf("Warning: lsc_interrupt needs to be off when "
2663                                 " using tx_first. Disabling.\n");
2664                 lsc_interrupt = 0;
2665         }
2666
2667         if (!nb_rxq && !nb_txq)
2668                 printf("Warning: Either rx or tx queues should be non-zero\n");
2669
2670         if (nb_rxq > 1 && nb_rxq > nb_txq)
2671                 printf("Warning: nb_rxq=%d enables RSS configuration, "
2672                        "but nb_txq=%d will prevent to fully test it.\n",
2673                        nb_rxq, nb_txq);
2674
2675         init_config();
2676
2677         if (hot_plug) {
2678                 /* enable hot plug monitoring */
2679                 ret = rte_dev_event_monitor_start();
2680                 if (ret) {
2681                         rte_errno = EINVAL;
2682                         return -1;
2683                 }
2684                 eth_dev_event_callback_register();
2685
2686         }
2687
2688         if (start_port(RTE_PORT_ALL) != 0)
2689                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
2690
2691         /* set all ports to promiscuous mode by default */
2692         RTE_ETH_FOREACH_DEV(port_id)
2693                 rte_eth_promiscuous_enable(port_id);
2694
2695         /* Init metrics library */
2696         rte_metrics_init(rte_socket_id());
2697
2698 #ifdef RTE_LIBRTE_LATENCY_STATS
2699         if (latencystats_enabled != 0) {
2700                 int ret = rte_latencystats_init(1, NULL);
2701                 if (ret)
2702                         printf("Warning: latencystats init()"
2703                                 " returned error %d\n", ret);
2704                 printf("Latencystats running on lcore %d\n",
2705                         latencystats_lcore_id);
2706         }
2707 #endif
2708
2709         /* Setup bitrate stats */
2710 #ifdef RTE_LIBRTE_BITRATE
2711         if (bitrate_enabled != 0) {
2712                 bitrate_data = rte_stats_bitrate_create();
2713                 if (bitrate_data == NULL)
2714                         rte_exit(EXIT_FAILURE,
2715                                 "Could not allocate bitrate data.\n");
2716                 rte_stats_bitrate_reg(bitrate_data);
2717         }
2718 #endif
2719
2720 #ifdef RTE_LIBRTE_CMDLINE
2721         if (strlen(cmdline_filename) != 0)
2722                 cmdline_read_from_file(cmdline_filename);
2723
2724         if (interactive == 1) {
2725                 if (auto_start) {
2726                         printf("Start automatic packet forwarding\n");
2727                         start_packet_forwarding(0);
2728                 }
2729                 prompt();
2730                 pmd_test_exit();
2731         } else
2732 #endif
2733         {
2734                 char c;
2735                 int rc;
2736
2737                 f_quit = 0;
2738
2739                 printf("No commandline core given, start packet forwarding\n");
2740                 start_packet_forwarding(tx_first);
2741                 if (stats_period != 0) {
2742                         uint64_t prev_time = 0, cur_time, diff_time = 0;
2743                         uint64_t timer_period;
2744
2745                         /* Convert to number of cycles */
2746                         timer_period = stats_period * rte_get_timer_hz();
2747
2748                         while (f_quit == 0) {
2749                                 cur_time = rte_get_timer_cycles();
2750                                 diff_time += cur_time - prev_time;
2751
2752                                 if (diff_time >= timer_period) {
2753                                         print_stats();
2754                                         /* Reset the timer */
2755                                         diff_time = 0;
2756                                 }
2757                                 /* Sleep to avoid unnecessary checks */
2758                                 prev_time = cur_time;
2759                                 sleep(1);
2760                         }
2761                 }
2762
2763                 printf("Press enter to exit\n");
2764                 rc = read(0, &c, 1);
2765                 pmd_test_exit();
2766                 if (rc < 0)
2767                         return 1;
2768         }
2769
2770         return 0;
2771 }