app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATE
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 uint16_t verbose_level = 0; /**< Silent by default. */
  67 int testpmd_logtype; /**< Log type for testpmd logs */
  68
  69 /* use master core for command line ? */
  70 uint8_t interactive = 0;
  71 uint8_t auto_start = 0;
  72 uint8_t tx_first;
  73 char cmdline_filename[PATH_MAX] = {0};
  74
  75 /*
  76  * NUMA support configuration.
  77  * When set, the NUMA support attempts to dispatch the allocation of the
  78  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  79  * probed ports among the CPU sockets 0 and 1.
  80  * Otherwise, all memory is allocated from CPU socket 0.
  81  */
  82 uint8_t numa_support = 1; /**< numa enabled by default */
  83
  84 /*
  85  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
  86  * not configured.
  87  */
  88 uint8_t socket_num = UMA_NO_CONFIG;
  89
  90 /*
  91  * Use ANONYMOUS mapped memory (might be not physically continuous) for mbufs.
  92  */
  93 uint8_t mp_anon = 0;
  94
  95 /*
  96  * Store specified sockets on which memory pool to be used by ports
  97  * is allocated.
  98  */
  99 uint8_t port_numa[RTE_MAX_ETHPORTS];
 100
 101 /*
 102  * Store specified sockets on which RX ring to be used by ports
 103  * is allocated.
 104  */
 105 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 106
 107 /*
 108  * Store specified sockets on which TX ring to be used by ports
 109  * is allocated.
 110  */
 111 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 112
 113 /*
 114  * Record the Ethernet address of peer target ports to which packets are
 115  * forwarded.
 116  * Must be instantiated with the ethernet addresses of peer traffic generator
 117  * ports.
 118  */
 119 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 120 portid_t nb_peer_eth_addrs = 0;
 121
 122 /*
 123  * Probed Target Environment.
 124  */
 125 struct rte_port *ports;        /**< For all probed ethernet ports. */
 126 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 127 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 128 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 129
 130 /*
 131  * Test Forwarding Configuration.
 132  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 133  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 134  */
 135 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 136 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 137 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 138 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 139
 140 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 141 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 142
 143 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 144 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 145
 146 /*
 147  * Forwarding engines.
 148  */
 149 struct fwd_engine * fwd_engines[] = {
 150         &io_fwd_engine,
 151         &mac_fwd_engine,
 152         &mac_swap_engine,
 153         &flow_gen_engine,
 154         &rx_only_engine,
 155         &tx_only_engine,
 156         &csum_fwd_engine,
 157         &icmp_echo_engine,
 158 #if defined RTE_LIBRTE_PMD_SOFTNIC
 159         &softnic_fwd_engine,
 160 #endif
 161 #ifdef RTE_LIBRTE_IEEE1588
 162         &ieee1588_fwd_engine,
 163 #endif
 164         NULL,
 165 };
 166
 167 struct fwd_config cur_fwd_config;
 168 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 169 uint32_t retry_enabled;
 170 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 171 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 172
 173 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 174 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 175                                       * specified on command-line. */
 176 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 177
 178 /*
 179  * In container, it cannot terminate the process which running with 'stats-period'
 180  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 181  */
 182 uint8_t f_quit;
 183
 184 /*
 185  * Configuration of packet segments used by the "txonly" processing engine.
 186  */
 187 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 188 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 189         TXONLY_DEF_PACKET_LEN,
 190 };
 191 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 192
 193 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 194 /**< Split policy for packets to TX. */
 195
 196 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 197 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 198
 199 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 200 uint8_t dcb_config = 0;
 201
 202 /* Whether the dcb is in testing status */
 203 uint8_t dcb_test = 0;
 204
 205 /*
 206  * Configurable number of RX/TX queues.
 207  */
 208 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 209 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 210
 211 /*
 212  * Configurable number of RX/TX ring descriptors.
 213  * Defaults are supplied by drivers via ethdev.
 214  */
 215 #define RTE_TEST_RX_DESC_DEFAULT 0
 216 #define RTE_TEST_TX_DESC_DEFAULT 0
 217 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 218 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 219
 220 #define RTE_PMD_PARAM_UNSET -1
 221 /*
 222  * Configurable values of RX and TX ring threshold registers.
 223  */
 224
 225 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 226 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 227 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 228
 229 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 230 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 231 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 232
 233 /*
 234  * Configurable value of RX free threshold.
 235  */
 236 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 237
 238 /*
 239  * Configurable value of RX drop enable.
 240  */
 241 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 242
 243 /*
 244  * Configurable value of TX free threshold.
 245  */
 246 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 247
 248 /*
 249  * Configurable value of TX RS bit threshold.
 250  */
 251 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 252
 253 /*
 254  * Receive Side Scaling (RSS) configuration.
 255  */
 256 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 257
 258 /*
 259  * Port topology configuration
 260  */
 261 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 262
 263 /*
 264  * Avoids to flush all the RX streams before starts forwarding.
 265  */
 266 uint8_t no_flush_rx = 0; /* flush by default */
 267
 268 /*
 269  * Flow API isolated mode.
 270  */
 271 uint8_t flow_isolate_all;
 272
 273 /*
 274  * Avoids to check link status when starting/stopping a port.
 275  */
 276 uint8_t no_link_check = 0; /* check by default */
 277
 278 /*
 279  * Enable link status change notification
 280  */
 281 uint8_t lsc_interrupt = 1; /* enabled by default */
 282
 283 /*
 284  * Enable device removal notification.
 285  */
 286 uint8_t rmv_interrupt = 1; /* enabled by default */
 287
 288 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 289
 290 /*
 291  * Display or mask ether events
 292  * Default to all events except VF_MBOX
 293  */
 294 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 295                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 296                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 297                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 298                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 299                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 300                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 301 /*
 302  * Decide if all memory are locked for performance.
 303  */
 304 int do_mlockall = 0;
 305
 306 /*
 307  * NIC bypass mode configuration options.
 308  */
 309
 310 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 311 /* The NIC bypass watchdog timeout. */
 312 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 313 #endif
 314
 315
 316 #ifdef RTE_LIBRTE_LATENCY_STATS
 317
 318 /*
 319  * Set when latency stats is enabled in the commandline
 320  */
 321 uint8_t latencystats_enabled;
 322
 323 /*
 324  * Lcore ID to serive latency statistics.
 325  */
 326 lcoreid_t latencystats_lcore_id = -1;
 327
 328 #endif
 329
 330 /*
 331  * Ethernet device configuration.
 332  */
 333 struct rte_eth_rxmode rx_mode = {
 334         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 335         .offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 336 };
 337
 338 struct rte_eth_txmode tx_mode = {
 339         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 340 };
 341
 342 struct rte_fdir_conf fdir_conf = {
 343         .mode = RTE_FDIR_MODE_NONE,
 344         .pballoc = RTE_FDIR_PBALLOC_64K,
 345         .status = RTE_FDIR_REPORT_STATUS,
 346         .mask = {
 347                 .vlan_tci_mask = 0xFFEF,
 348                 .ipv4_mask     = {
 349                         .src_ip = 0xFFFFFFFF,
 350                         .dst_ip = 0xFFFFFFFF,
 351                 },
 352                 .ipv6_mask     = {
 353                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 354                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 355                 },
 356                 .src_port_mask = 0xFFFF,
 357                 .dst_port_mask = 0xFFFF,
 358                 .mac_addr_byte_mask = 0xFF,
 359                 .tunnel_type_mask = 1,
 360                 .tunnel_id_mask = 0xFFFFFFFF,
 361         },
 362         .drop_queue = 127,
 363 };
 364
 365 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 366
 367 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 368 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 369
 370 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 371 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 372
 373 uint16_t nb_tx_queue_stats_mappings = 0;
 374 uint16_t nb_rx_queue_stats_mappings = 0;
 375
 376 /*
 377  * Display zero values by default for xstats
 378  */
 379 uint8_t xstats_hide_zero;
 380
 381 unsigned int num_sockets = 0;
 382 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 383
 384 #ifdef RTE_LIBRTE_BITRATE
 385 /* Bitrate statistics */
 386 struct rte_stats_bitrates *bitrate_data;
 387 lcoreid_t bitrate_lcore_id;
 388 uint8_t bitrate_enabled;
 389 #endif
 390
 391 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 392 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 393
 394 struct vxlan_encap_conf vxlan_encap_conf = {
 395         .select_ipv4 = 1,
 396         .select_vlan = 0,
 397         .vni = "\x00\x00\x00",
 398         .udp_src = 0,
 399         .udp_dst = RTE_BE16(4789),
 400         .ipv4_src = IPv4(127, 0, 0, 1),
 401         .ipv4_dst = IPv4(255, 255, 255, 255),
 402         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 403                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 404         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 405                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 406         .vlan_tci = 0,
 407         .eth_src = "\x00\x00\x00\x00\x00\x00",
 408         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 409 };
 410
 411 struct nvgre_encap_conf nvgre_encap_conf = {
 412         .select_ipv4 = 1,
 413         .select_vlan = 0,
 414         .tni = "\x00\x00\x00",
 415         .ipv4_src = IPv4(127, 0, 0, 1),
 416         .ipv4_dst = IPv4(255, 255, 255, 255),
 417         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 418                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 419         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 420                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 421         .vlan_tci = 0,
 422         .eth_src = "\x00\x00\x00\x00\x00\x00",
 423         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 424 };
 425
 426 /* Forward function declarations */
 427 static void map_port_queue_stats_mapping_registers(portid_t pi,
 428                                                    struct rte_port *port);
 429 static void check_all_ports_link_status(uint32_t port_mask);
 430 static int eth_event_callback(portid_t port_id,
 431                               enum rte_eth_event_type type,
 432                               void *param, void *ret_param);
 433 static void eth_dev_event_callback(char *device_name,
 434                                 enum rte_dev_event_type type,
 435                                 void *param);
 436 static int eth_dev_event_callback_register(void);
 437 static int eth_dev_event_callback_unregister(void);
 438
 439
 440 /*
 441  * Check if all the ports are started.
 442  * If yes, return positive value. If not, return zero.
 443  */
 444 static int all_ports_started(void);
 445
 446 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 447 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN;
 448
 449 /*
 450  * Helper function to check if socket is already discovered.
 451  * If yes, return positive value. If not, return zero.
 452  */
 453 int
 454 new_socket_id(unsigned int socket_id)
 455 {
 456         unsigned int i;
 457
 458         for (i = 0; i < num_sockets; i++) {
 459                 if (socket_ids[i] == socket_id)
 460                         return 0;
 461         }
 462         return 1;
 463 }
 464
 465 /*
 466  * Setup default configuration.
 467  */
 468 static void
 469 set_default_fwd_lcores_config(void)
 470 {
 471         unsigned int i;
 472         unsigned int nb_lc;
 473         unsigned int sock_num;
 474
 475         nb_lc = 0;
 476         for (i = 0; i < RTE_MAX_LCORE; i++) {
 477                 sock_num = rte_lcore_to_socket_id(i);
 478                 if (new_socket_id(sock_num)) {
 479                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 480                                 rte_exit(EXIT_FAILURE,
 481                                          "Total sockets greater than %u\n",
 482                                          RTE_MAX_NUMA_NODES);
 483                         }
 484                         socket_ids[num_sockets++] = sock_num;
 485                 }
 486                 if (!rte_lcore_is_enabled(i))
 487                         continue;
 488                 if (i == rte_get_master_lcore())
 489                         continue;
 490                 fwd_lcores_cpuids[nb_lc++] = i;
 491         }
 492         nb_lcores = (lcoreid_t) nb_lc;
 493         nb_cfg_lcores = nb_lcores;
 494         nb_fwd_lcores = 1;
 495 }
 496
 497 static void
 498 set_def_peer_eth_addrs(void)
 499 {
 500         portid_t i;
 501
 502         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 503                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 504                 peer_eth_addrs[i].addr_bytes[5] = i;
 505         }
 506 }
 507
 508 static void
 509 set_default_fwd_ports_config(void)
 510 {
 511         portid_t pt_id;
 512         int i = 0;
 513
 514         RTE_ETH_FOREACH_DEV(pt_id)
 515                 fwd_ports_ids[i++] = pt_id;
 516
 517         nb_cfg_ports = nb_ports;
 518         nb_fwd_ports = nb_ports;
 519 }
 520
 521 void
 522 set_def_fwd_config(void)
 523 {
 524         set_default_fwd_lcores_config();
 525         set_def_peer_eth_addrs();
 526         set_default_fwd_ports_config();
 527 }
 528
 529 /*
 530  * Configuration initialisation done once at init time.
 531  */
 532 static void
 533 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 534                  unsigned int socket_id)
 535 {
 536         char pool_name[RTE_MEMPOOL_NAMESIZE];
 537         struct rte_mempool *rte_mp = NULL;
 538         uint32_t mb_size;
 539
 540         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 541         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 542
 543         TESTPMD_LOG(INFO,
 544                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 545                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 546
 547         if (mp_anon != 0) {
 548                 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 549                         mb_size, (unsigned) mb_mempool_cache,
 550                         sizeof(struct rte_pktmbuf_pool_private),
 551                         socket_id, 0);
 552                 if (rte_mp == NULL)
 553                         goto err;
 554
 555                 if (rte_mempool_populate_anon(rte_mp) == 0) {
 556                         rte_mempool_free(rte_mp);
 557                         rte_mp = NULL;
 558                         goto err;
 559                 }
 560                 rte_pktmbuf_pool_init(rte_mp, NULL);
 561                 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 562         } else {
 563                 /* wrapper to rte_mempool_create() */
 564                 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 565                                 rte_mbuf_best_mempool_ops());
 566                 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 567                         mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 568         }
 569
 570 err:
 571         if (rte_mp == NULL) {
 572                 rte_exit(EXIT_FAILURE,
 573                         "Creation of mbuf pool for socket %u failed: %s\n",
 574                         socket_id, rte_strerror(rte_errno));
 575         } else if (verbose_level > 0) {
 576                 rte_mempool_dump(stdout, rte_mp);
 577         }
 578 }
 579
 580 /*
 581  * Check given socket id is valid or not with NUMA mode,
 582  * if valid, return 0, else return -1
 583  */
 584 static int
 585 check_socket_id(const unsigned int socket_id)
 586 {
 587         static int warning_once = 0;
 588
 589         if (new_socket_id(socket_id)) {
 590                 if (!warning_once && numa_support)
 591                         printf("Warning: NUMA should be configured manually by"
 592                                " using --port-numa-config and"
 593                                " --ring-numa-config parameters along with"
 594                                " --numa.\n");
 595                 warning_once = 1;
 596                 return -1;
 597         }
 598         return 0;
 599 }
 600
 601 /*
 602  * Get the allowed maximum number of RX queues.
 603  * *pid return the port id which has minimal value of
 604  * max_rx_queues in all ports.
 605  */
 606 queueid_t
 607 get_allowed_max_nb_rxq(portid_t *pid)
 608 {
 609         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
 610         portid_t pi;
 611         struct rte_eth_dev_info dev_info;
 612
 613         RTE_ETH_FOREACH_DEV(pi) {
 614                 rte_eth_dev_info_get(pi, &dev_info);
 615                 if (dev_info.max_rx_queues < allowed_max_rxq) {
 616                         allowed_max_rxq = dev_info.max_rx_queues;
 617                         *pid = pi;
 618                 }
 619         }
 620         return allowed_max_rxq;
 621 }
 622
 623 /*
 624  * Check input rxq is valid or not.
 625  * If input rxq is not greater than any of maximum number
 626  * of RX queues of all ports, it is valid.
 627  * if valid, return 0, else return -1
 628  */
 629 int
 630 check_nb_rxq(queueid_t rxq)
 631 {
 632         queueid_t allowed_max_rxq;
 633         portid_t pid = 0;
 634
 635         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
 636         if (rxq > allowed_max_rxq) {
 637                 printf("Fail: input rxq (%u) can't be greater "
 638                        "than max_rx_queues (%u) of port %u\n",
 639                        rxq,
 640                        allowed_max_rxq,
 641                        pid);
 642                 return -1;
 643         }
 644         return 0;
 645 }
 646
 647 /*
 648  * Get the allowed maximum number of TX queues.
 649  * *pid return the port id which has minimal value of
 650  * max_tx_queues in all ports.
 651  */
 652 queueid_t
 653 get_allowed_max_nb_txq(portid_t *pid)
 654 {
 655         queueid_t allowed_max_txq = MAX_QUEUE_ID;
 656         portid_t pi;
 657         struct rte_eth_dev_info dev_info;
 658
 659         RTE_ETH_FOREACH_DEV(pi) {
 660                 rte_eth_dev_info_get(pi, &dev_info);
 661                 if (dev_info.max_tx_queues < allowed_max_txq) {
 662                         allowed_max_txq = dev_info.max_tx_queues;
 663                         *pid = pi;
 664                 }
 665         }
 666         return allowed_max_txq;
 667 }
 668
 669 /*
 670  * Check input txq is valid or not.
 671  * If input txq is not greater than any of maximum number
 672  * of TX queues of all ports, it is valid.
 673  * if valid, return 0, else return -1
 674  */
 675 int
 676 check_nb_txq(queueid_t txq)
 677 {
 678         queueid_t allowed_max_txq;
 679         portid_t pid = 0;
 680
 681         allowed_max_txq = get_allowed_max_nb_txq(&pid);
 682         if (txq > allowed_max_txq) {
 683                 printf("Fail: input txq (%u) can't be greater "
 684                        "than max_tx_queues (%u) of port %u\n",
 685                        txq,
 686                        allowed_max_txq,
 687                        pid);
 688                 return -1;
 689         }
 690         return 0;
 691 }
 692
 693 static void
 694 init_config(void)
 695 {
 696         portid_t pid;
 697         struct rte_port *port;
 698         struct rte_mempool *mbp;
 699         unsigned int nb_mbuf_per_pool;
 700         lcoreid_t  lc_id;
 701         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
 702         struct rte_gro_param gro_param;
 703         uint32_t gso_types;
 704         int k;
 705
 706         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
 707
 708         if (numa_support) {
 709                 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
 710                 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
 711                 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
 712         }
 713
 714         /* Configuration of logical cores. */
 715         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
 716                                 sizeof(struct fwd_lcore *) * nb_lcores,
 717                                 RTE_CACHE_LINE_SIZE);
 718         if (fwd_lcores == NULL) {
 719                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
 720                                                         "failed\n", nb_lcores);
 721         }
 722         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 723                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
 724                                                sizeof(struct fwd_lcore),
 725                                                RTE_CACHE_LINE_SIZE);
 726                 if (fwd_lcores[lc_id] == NULL) {
 727                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
 728                                                                 "failed\n");
 729                 }
 730                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
 731         }
 732
 733         RTE_ETH_FOREACH_DEV(pid) {
 734                 port = &ports[pid];
 735                 /* Apply default TxRx configuration for all ports */
 736                 port->dev_conf.txmode = tx_mode;
 737                 port->dev_conf.rxmode = rx_mode;
 738                 rte_eth_dev_info_get(pid, &port->dev_info);
 739
 740                 if (!(port->dev_info.rx_offload_capa &
 741                                         DEV_RX_OFFLOAD_CRC_STRIP))
 742                         port->dev_conf.rxmode.offloads &=
 743                                 ~DEV_RX_OFFLOAD_CRC_STRIP;
 744                 if (!(port->dev_info.tx_offload_capa &
 745                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
 746                         port->dev_conf.txmode.offloads &=
 747                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 748                 if (numa_support) {
 749                         if (port_numa[pid] != NUMA_NO_CONFIG)
 750                                 port_per_socket[port_numa[pid]]++;
 751                         else {
 752                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
 753
 754                                 /* if socket_id is invalid, set to 0 */
 755                                 if (check_socket_id(socket_id) < 0)
 756                                         socket_id = 0;
 757                                 port_per_socket[socket_id]++;
 758                         }
 759                 }
 760
 761                 /* Apply Rx offloads configuration */
 762                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
 763                         port->rx_conf[k].offloads =
 764                                 port->dev_conf.rxmode.offloads;
 765                 /* Apply Tx offloads configuration */
 766                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
 767                         port->tx_conf[k].offloads =
 768                                 port->dev_conf.txmode.offloads;
 769
 770                 /* set flag to initialize port/queue */
 771                 port->need_reconfig = 1;
 772                 port->need_reconfig_queues = 1;
 773         }
 774
 775         /*
 776          * Create pools of mbuf.
 777          * If NUMA support is disabled, create a single pool of mbuf in
 778          * socket 0 memory by default.
 779          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
 780          *
 781          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
 782          * nb_txd can be configured at run time.
 783          */
 784         if (param_total_num_mbufs)
 785                 nb_mbuf_per_pool = param_total_num_mbufs;
 786         else {
 787                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
 788                         (nb_lcores * mb_mempool_cache) +
 789                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
 790                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
 791         }
 792
 793         if (numa_support) {
 794                 uint8_t i;
 795
 796                 for (i = 0; i < num_sockets; i++)
 797                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
 798                                          socket_ids[i]);
 799         } else {
 800                 if (socket_num == UMA_NO_CONFIG)
 801                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0);
 802                 else
 803                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
 804                                                  socket_num);
 805         }
 806
 807         init_port_config();
 808
 809         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
 810                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
 811         /*
 812          * Records which Mbuf pool to use by each logical core, if needed.
 813          */
 814         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 815                 mbp = mbuf_pool_find(
 816                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
 817
 818                 if (mbp == NULL)
 819                         mbp = mbuf_pool_find(0);
 820                 fwd_lcores[lc_id]->mbp = mbp;
 821                 /* initialize GSO context */
 822                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
 823                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
 824                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
 825                 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN -
 826                         ETHER_CRC_LEN;
 827                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
 828         }
 829
 830         /* Configuration of packet forwarding streams. */
 831         if (init_fwd_streams() < 0)
 832                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 833
 834         fwd_config_setup();
 835
 836         /* create a gro context for each lcore */
 837         gro_param.gro_types = RTE_GRO_TCP_IPV4;
 838         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
 839         gro_param.max_item_per_flow = MAX_PKT_BURST;
 840         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
 841                 gro_param.socket_id = rte_lcore_to_socket_id(
 842                                 fwd_lcores_cpuids[lc_id]);
 843                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
 844                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
 845                         rte_exit(EXIT_FAILURE,
 846                                         "rte_gro_ctx_create() failed\n");
 847                 }
 848         }
 849
 850 #if defined RTE_LIBRTE_PMD_SOFTNIC
 851         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
 852                 RTE_ETH_FOREACH_DEV(pid) {
 853                         port = &ports[pid];
 854                         const char *driver = port->dev_info.driver_name;
 855
 856                         if (strcmp(driver, "net_softnic") == 0)
 857                                 port->softport.fwd_lcore_arg = fwd_lcores;
 858                 }
 859         }
 860 #endif
 861
 862 }
 863
 864
 865 void
 866 reconfig(portid_t new_port_id, unsigned socket_id)
 867 {
 868         struct rte_port *port;
 869
 870         /* Reconfiguration of Ethernet ports. */
 871         port = &ports[new_port_id];
 872         rte_eth_dev_info_get(new_port_id, &port->dev_info);
 873
 874         /* set flag to initialize port/queue */
 875         port->need_reconfig = 1;
 876         port->need_reconfig_queues = 1;
 877         port->socket_id = socket_id;
 878
 879         init_port_config();
 880 }
 881
 882
 883 int
 884 init_fwd_streams(void)
 885 {
 886         portid_t pid;
 887         struct rte_port *port;
 888         streamid_t sm_id, nb_fwd_streams_new;
 889         queueid_t q;
 890
 891         /* set socket id according to numa or not */
 892         RTE_ETH_FOREACH_DEV(pid) {
 893                 port = &ports[pid];
 894                 if (nb_rxq > port->dev_info.max_rx_queues) {
 895                         printf("Fail: nb_rxq(%d) is greater than "
 896                                 "max_rx_queues(%d)\n", nb_rxq,
 897                                 port->dev_info.max_rx_queues);
 898                         return -1;
 899                 }
 900                 if (nb_txq > port->dev_info.max_tx_queues) {
 901                         printf("Fail: nb_txq(%d) is greater than "
 902                                 "max_tx_queues(%d)\n", nb_txq,
 903                                 port->dev_info.max_tx_queues);
 904                         return -1;
 905                 }
 906                 if (numa_support) {
 907                         if (port_numa[pid] != NUMA_NO_CONFIG)
 908                                 port->socket_id = port_numa[pid];
 909                         else {
 910                                 port->socket_id = rte_eth_dev_socket_id(pid);
 911
 912                                 /* if socket_id is invalid, set to 0 */
 913                                 if (check_socket_id(port->socket_id) < 0)
 914                                         port->socket_id = 0;
 915                         }
 916                 }
 917                 else {
 918                         if (socket_num == UMA_NO_CONFIG)
 919                                 port->socket_id = 0;
 920                         else
 921                                 port->socket_id = socket_num;
 922                 }
 923         }
 924
 925         q = RTE_MAX(nb_rxq, nb_txq);
 926         if (q == 0) {
 927                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
 928                 return -1;
 929         }
 930         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
 931         if (nb_fwd_streams_new == nb_fwd_streams)
 932                 return 0;
 933         /* clear the old */
 934         if (fwd_streams != NULL) {
 935                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 936                         if (fwd_streams[sm_id] == NULL)
 937                                 continue;
 938                         rte_free(fwd_streams[sm_id]);
 939                         fwd_streams[sm_id] = NULL;
 940                 }
 941                 rte_free(fwd_streams);
 942                 fwd_streams = NULL;
 943         }
 944
 945         /* init new */
 946         nb_fwd_streams = nb_fwd_streams_new;
 947         if (nb_fwd_streams) {
 948                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
 949                         sizeof(struct fwd_stream *) * nb_fwd_streams,
 950                         RTE_CACHE_LINE_SIZE);
 951                 if (fwd_streams == NULL)
 952                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
 953                                  " (struct fwd_stream *)) failed\n",
 954                                  nb_fwd_streams);
 955
 956                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
 957                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
 958                                 " struct fwd_stream", sizeof(struct fwd_stream),
 959                                 RTE_CACHE_LINE_SIZE);
 960                         if (fwd_streams[sm_id] == NULL)
 961                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
 962                                          "(struct fwd_stream) failed\n");
 963                 }
 964         }
 965
 966         return 0;
 967 }
 968
 969 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 970 static void
 971 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
 972 {
 973         unsigned int total_burst;
 974         unsigned int nb_burst;
 975         unsigned int burst_stats[3];
 976         uint16_t pktnb_stats[3];
 977         uint16_t nb_pkt;
 978         int burst_percent[3];
 979
 980         /*
 981          * First compute the total number of packet bursts and the
 982          * two highest numbers of bursts of the same number of packets.
 983          */
 984         total_burst = 0;
 985         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
 986         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
 987         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
 988                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
 989                 if (nb_burst == 0)
 990                         continue;
 991                 total_burst += nb_burst;
 992                 if (nb_burst > burst_stats[0]) {
 993                         burst_stats[1] = burst_stats[0];
 994                         pktnb_stats[1] = pktnb_stats[0];
 995                         burst_stats[0] = nb_burst;
 996                         pktnb_stats[0] = nb_pkt;
 997                 } else if (nb_burst > burst_stats[1]) {
 998                         burst_stats[1] = nb_burst;
 999                         pktnb_stats[1] = nb_pkt;
1000                 }
1001         }
1002         if (total_burst == 0)
1003                 return;
1004         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1005         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1006                burst_percent[0], (int) pktnb_stats[0]);
1007         if (burst_stats[0] == total_burst) {
1008                 printf("]\n");
1009                 return;
1010         }
1011         if (burst_stats[0] + burst_stats[1] == total_burst) {
1012                 printf(" + %d%% of %d pkts]\n",
1013                        100 - burst_percent[0], pktnb_stats[1]);
1014                 return;
1015         }
1016         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1017         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1018         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1019                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1020                 return;
1021         }
1022         printf(" + %d%% of %d pkts + %d%% of others]\n",
1023                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1024 }
1025 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1026
1027 static void
1028 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats)
1029 {
1030         struct rte_port *port;
1031         uint8_t i;
1032
1033         static const char *fwd_stats_border = "----------------------";
1034
1035         port = &ports[port_id];
1036         printf("\n  %s Forward statistics for port %-2d %s\n",
1037                fwd_stats_border, port_id, fwd_stats_border);
1038
1039         if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) {
1040                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1041                        "%-"PRIu64"\n",
1042                        stats->ipackets, stats->imissed,
1043                        (uint64_t) (stats->ipackets + stats->imissed));
1044
1045                 if (cur_fwd_eng == &csum_fwd_engine)
1046                         printf("  Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64" \n",
1047                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
1048                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
1049                         printf("  RX-error: %-"PRIu64"\n",  stats->ierrors);
1050                         printf("  RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf);
1051                 }
1052
1053                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1054                        "%-"PRIu64"\n",
1055                        stats->opackets, port->tx_dropped,
1056                        (uint64_t) (stats->opackets + port->tx_dropped));
1057         }
1058         else {
1059                 printf("  RX-packets:             %14"PRIu64"    RX-dropped:%14"PRIu64"    RX-total:"
1060                        "%14"PRIu64"\n",
1061                        stats->ipackets, stats->imissed,
1062                        (uint64_t) (stats->ipackets + stats->imissed));
1063
1064                 if (cur_fwd_eng == &csum_fwd_engine)
1065                         printf("  Bad-ipcsum:%14"PRIu64"    Bad-l4csum:%14"PRIu64"\n",
1066                                port->rx_bad_ip_csum, port->rx_bad_l4_csum);
1067                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
1068                         printf("  RX-error:%"PRIu64"\n", stats->ierrors);
1069                         printf("  RX-nombufs:             %14"PRIu64"\n",
1070                                stats->rx_nombuf);
1071                 }
1072
1073                 printf("  TX-packets:             %14"PRIu64"    TX-dropped:%14"PRIu64"    TX-total:"
1074                        "%14"PRIu64"\n",
1075                        stats->opackets, port->tx_dropped,
1076                        (uint64_t) (stats->opackets + port->tx_dropped));
1077         }
1078
1079 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1080         if (port->rx_stream)
1081                 pkt_burst_stats_display("RX",
1082                         &port->rx_stream->rx_burst_stats);
1083         if (port->tx_stream)
1084                 pkt_burst_stats_display("TX",
1085                         &port->tx_stream->tx_burst_stats);
1086 #endif
1087
1088         if (port->rx_queue_stats_mapping_enabled) {
1089                 printf("\n");
1090                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
1091                         printf("  Stats reg %2d RX-packets:%14"PRIu64
1092                                "     RX-errors:%14"PRIu64
1093                                "    RX-bytes:%14"PRIu64"\n",
1094                                i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]);
1095                 }
1096                 printf("\n");
1097         }
1098         if (port->tx_queue_stats_mapping_enabled) {
1099                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
1100                         printf("  Stats reg %2d TX-packets:%14"PRIu64
1101                                "                                 TX-bytes:%14"PRIu64"\n",
1102                                i, stats->q_opackets[i], stats->q_obytes[i]);
1103                 }
1104         }
1105
1106         printf("  %s--------------------------------%s\n",
1107                fwd_stats_border, fwd_stats_border);
1108 }
1109
1110 static void
1111 fwd_stream_stats_display(streamid_t stream_id)
1112 {
1113         struct fwd_stream *fs;
1114         static const char *fwd_top_stats_border = "-------";
1115
1116         fs = fwd_streams[stream_id];
1117         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1118             (fs->fwd_dropped == 0))
1119                 return;
1120         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1121                "TX Port=%2d/Queue=%2d %s\n",
1122                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1123                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1124         printf("  RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u",
1125                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1126
1127         /* if checksum mode */
1128         if (cur_fwd_eng == &csum_fwd_engine) {
1129                printf("  RX- bad IP checksum: %-14u  Rx- bad L4 checksum: "
1130                         "%-14u\n", fs->rx_bad_ip_csum, fs->rx_bad_l4_csum);
1131         }
1132
1133 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1134         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1135         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1136 #endif
1137 }
1138
1139 static void
1140 flush_fwd_rx_queues(void)
1141 {
1142         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1143         portid_t  rxp;
1144         portid_t port_id;
1145         queueid_t rxq;
1146         uint16_t  nb_rx;
1147         uint16_t  i;
1148         uint8_t   j;
1149         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1150         uint64_t timer_period;
1151
1152         /* convert to number of cycles */
1153         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1154
1155         for (j = 0; j < 2; j++) {
1156                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1157                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1158                                 port_id = fwd_ports_ids[rxp];
1159                                 /**
1160                                 * testpmd can stuck in the below do while loop
1161                                 * if rte_eth_rx_burst() always returns nonzero
1162                                 * packets. So timer is added to exit this loop
1163                                 * after 1sec timer expiry.
1164                                 */
1165                                 prev_tsc = rte_rdtsc();
1166                                 do {
1167                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1168                                                 pkts_burst, MAX_PKT_BURST);
1169                                         for (i = 0; i < nb_rx; i++)
1170                                                 rte_pktmbuf_free(pkts_burst[i]);
1171
1172                                         cur_tsc = rte_rdtsc();
1173                                         diff_tsc = cur_tsc - prev_tsc;
1174                                         timer_tsc += diff_tsc;
1175                                 } while ((nb_rx > 0) &&
1176                                         (timer_tsc < timer_period));
1177                                 timer_tsc = 0;
1178                         }
1179                 }
1180                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1181         }
1182 }
1183
1184 static void
1185 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1186 {
1187         struct fwd_stream **fsm;
1188         streamid_t nb_fs;
1189         streamid_t sm_id;
1190 #ifdef RTE_LIBRTE_BITRATE
1191         uint64_t tics_per_1sec;
1192         uint64_t tics_datum;
1193         uint64_t tics_current;
1194         uint16_t idx_port;
1195
1196         tics_datum = rte_rdtsc();
1197         tics_per_1sec = rte_get_timer_hz();
1198 #endif
1199         fsm = &fwd_streams[fc->stream_idx];
1200         nb_fs = fc->stream_nb;
1201         do {
1202                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1203                         (*pkt_fwd)(fsm[sm_id]);
1204 #ifdef RTE_LIBRTE_BITRATE
1205                 if (bitrate_enabled != 0 &&
1206                                 bitrate_lcore_id == rte_lcore_id()) {
1207                         tics_current = rte_rdtsc();
1208                         if (tics_current - tics_datum >= tics_per_1sec) {
1209                                 /* Periodic bitrate calculation */
1210                                 RTE_ETH_FOREACH_DEV(idx_port)
1211                                         rte_stats_bitrate_calc(bitrate_data,
1212                                                 idx_port);
1213                                 tics_datum = tics_current;
1214                         }
1215                 }
1216 #endif
1217 #ifdef RTE_LIBRTE_LATENCY_STATS
1218                 if (latencystats_enabled != 0 &&
1219                                 latencystats_lcore_id == rte_lcore_id())
1220                         rte_latencystats_update();
1221 #endif
1222
1223         } while (! fc->stopped);
1224 }
1225
1226 static int
1227 start_pkt_forward_on_core(void *fwd_arg)
1228 {
1229         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1230                              cur_fwd_config.fwd_eng->packet_fwd);
1231         return 0;
1232 }
1233
1234 /*
1235  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1236  * Used to start communication flows in network loopback test configurations.
1237  */
1238 static int
1239 run_one_txonly_burst_on_core(void *fwd_arg)
1240 {
1241         struct fwd_lcore *fwd_lc;
1242         struct fwd_lcore tmp_lcore;
1243
1244         fwd_lc = (struct fwd_lcore *) fwd_arg;
1245         tmp_lcore = *fwd_lc;
1246         tmp_lcore.stopped = 1;
1247         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1248         return 0;
1249 }
1250
1251 /*
1252  * Launch packet forwarding:
1253  *     - Setup per-port forwarding context.
1254  *     - launch logical cores with their forwarding configuration.
1255  */
1256 static void
1257 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1258 {
1259         port_fwd_begin_t port_fwd_begin;
1260         unsigned int i;
1261         unsigned int lc_id;
1262         int diag;
1263
1264         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1265         if (port_fwd_begin != NULL) {
1266                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1267                         (*port_fwd_begin)(fwd_ports_ids[i]);
1268         }
1269         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1270                 lc_id = fwd_lcores_cpuids[i];
1271                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1272                         fwd_lcores[i]->stopped = 0;
1273                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1274                                                      fwd_lcores[i], lc_id);
1275                         if (diag != 0)
1276                                 printf("launch lcore %u failed - diag=%d\n",
1277                                        lc_id, diag);
1278                 }
1279         }
1280 }
1281
1282 /*
1283  * Update the forward ports list.
1284  */
1285 void
1286 update_fwd_ports(portid_t new_pid)
1287 {
1288         unsigned int i;
1289         unsigned int new_nb_fwd_ports = 0;
1290         int move = 0;
1291
1292         for (i = 0; i < nb_fwd_ports; ++i) {
1293                 if (port_id_is_invalid(fwd_ports_ids[i], DISABLED_WARN))
1294                         move = 1;
1295                 else if (move)
1296                         fwd_ports_ids[new_nb_fwd_ports++] = fwd_ports_ids[i];
1297                 else
1298                         new_nb_fwd_ports++;
1299         }
1300         if (new_pid < RTE_MAX_ETHPORTS)
1301                 fwd_ports_ids[new_nb_fwd_ports++] = new_pid;
1302
1303         nb_fwd_ports = new_nb_fwd_ports;
1304         nb_cfg_ports = new_nb_fwd_ports;
1305 }
1306
1307 /*
1308  * Launch packet forwarding configuration.
1309  */
1310 void
1311 start_packet_forwarding(int with_tx_first)
1312 {
1313         port_fwd_begin_t port_fwd_begin;
1314         port_fwd_end_t  port_fwd_end;
1315         struct rte_port *port;
1316         unsigned int i;
1317         portid_t   pt_id;
1318         streamid_t sm_id;
1319
1320         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1321                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1322
1323         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1324                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1325
1326         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1327                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1328                 (!nb_rxq || !nb_txq))
1329                 rte_exit(EXIT_FAILURE,
1330                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1331                         cur_fwd_eng->fwd_mode_name);
1332
1333         if (all_ports_started() == 0) {
1334                 printf("Not all ports were started\n");
1335                 return;
1336         }
1337         if (test_done == 0) {
1338                 printf("Packet forwarding already started\n");
1339                 return;
1340         }
1341
1342
1343         if(dcb_test) {
1344                 for (i = 0; i < nb_fwd_ports; i++) {
1345                         pt_id = fwd_ports_ids[i];
1346                         port = &ports[pt_id];
1347                         if (!port->dcb_flag) {
1348                                 printf("In DCB mode, all forwarding ports must "
1349                                        "be configured in this mode.\n");
1350                                 return;
1351                         }
1352                 }
1353                 if (nb_fwd_lcores == 1) {
1354                         printf("In DCB mode,the nb forwarding cores "
1355                                "should be larger than 1.\n");
1356                         return;
1357                 }
1358         }
1359         test_done = 0;
1360
1361         fwd_config_setup();
1362
1363         if(!no_flush_rx)
1364                 flush_fwd_rx_queues();
1365
1366         pkt_fwd_config_display(&cur_fwd_config);
1367         rxtx_config_display();
1368
1369         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1370                 pt_id = fwd_ports_ids[i];
1371                 port = &ports[pt_id];
1372                 rte_eth_stats_get(pt_id, &port->stats);
1373                 port->tx_dropped = 0;
1374
1375                 map_port_queue_stats_mapping_registers(pt_id, port);
1376         }
1377         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1378                 fwd_streams[sm_id]->rx_packets = 0;
1379                 fwd_streams[sm_id]->tx_packets = 0;
1380                 fwd_streams[sm_id]->fwd_dropped = 0;
1381                 fwd_streams[sm_id]->rx_bad_ip_csum = 0;
1382                 fwd_streams[sm_id]->rx_bad_l4_csum = 0;
1383
1384 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1385                 memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
1386                        sizeof(fwd_streams[sm_id]->rx_burst_stats));
1387                 memset(&fwd_streams[sm_id]->tx_burst_stats, 0,
1388                        sizeof(fwd_streams[sm_id]->tx_burst_stats));
1389 #endif
1390 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1391                 fwd_streams[sm_id]->core_cycles = 0;
1392 #endif
1393         }
1394         if (with_tx_first) {
1395                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1396                 if (port_fwd_begin != NULL) {
1397                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1398                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1399                 }
1400                 while (with_tx_first--) {
1401                         launch_packet_forwarding(
1402                                         run_one_txonly_burst_on_core);
1403                         rte_eal_mp_wait_lcore();
1404                 }
1405                 port_fwd_end = tx_only_engine.port_fwd_end;
1406                 if (port_fwd_end != NULL) {
1407                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1408                                 (*port_fwd_end)(fwd_ports_ids[i]);
1409                 }
1410         }
1411         launch_packet_forwarding(start_pkt_forward_on_core);
1412 }
1413
1414 void
1415 stop_packet_forwarding(void)
1416 {
1417         struct rte_eth_stats stats;
1418         struct rte_port *port;
1419         port_fwd_end_t  port_fwd_end;
1420         int i;
1421         portid_t   pt_id;
1422         streamid_t sm_id;
1423         lcoreid_t  lc_id;
1424         uint64_t total_recv;
1425         uint64_t total_xmit;
1426         uint64_t total_rx_dropped;
1427         uint64_t total_tx_dropped;
1428         uint64_t total_rx_nombuf;
1429         uint64_t tx_dropped;
1430         uint64_t rx_bad_ip_csum;
1431         uint64_t rx_bad_l4_csum;
1432 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1433         uint64_t fwd_cycles;
1434 #endif
1435
1436         static const char *acc_stats_border = "+++++++++++++++";
1437
1438         if (test_done) {
1439                 printf("Packet forwarding not started\n");
1440                 return;
1441         }
1442         printf("Telling cores to stop...");
1443         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1444                 fwd_lcores[lc_id]->stopped = 1;
1445         printf("\nWaiting for lcores to finish...\n");
1446         rte_eal_mp_wait_lcore();
1447         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1448         if (port_fwd_end != NULL) {
1449                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1450                         pt_id = fwd_ports_ids[i];
1451                         (*port_fwd_end)(pt_id);
1452                 }
1453         }
1454 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1455         fwd_cycles = 0;
1456 #endif
1457         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1458                 if (cur_fwd_config.nb_fwd_streams >
1459                     cur_fwd_config.nb_fwd_ports) {
1460                         fwd_stream_stats_display(sm_id);
1461                         ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL;
1462                         ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL;
1463                 } else {
1464                         ports[fwd_streams[sm_id]->tx_port].tx_stream =
1465                                 fwd_streams[sm_id];
1466                         ports[fwd_streams[sm_id]->rx_port].rx_stream =
1467                                 fwd_streams[sm_id];
1468                 }
1469                 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped;
1470                 tx_dropped = (uint64_t) (tx_dropped +
1471                                          fwd_streams[sm_id]->fwd_dropped);
1472                 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped;
1473
1474                 rx_bad_ip_csum =
1475                         ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum;
1476                 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum +
1477                                          fwd_streams[sm_id]->rx_bad_ip_csum);
1478                 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum =
1479                                                         rx_bad_ip_csum;
1480
1481                 rx_bad_l4_csum =
1482                         ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum;
1483                 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum +
1484                                          fwd_streams[sm_id]->rx_bad_l4_csum);
1485                 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum =
1486                                                         rx_bad_l4_csum;
1487
1488 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1489                 fwd_cycles = (uint64_t) (fwd_cycles +
1490                                          fwd_streams[sm_id]->core_cycles);
1491 #endif
1492         }
1493         total_recv = 0;
1494         total_xmit = 0;
1495         total_rx_dropped = 0;
1496         total_tx_dropped = 0;
1497         total_rx_nombuf  = 0;
1498         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1499                 pt_id = fwd_ports_ids[i];
1500
1501                 port = &ports[pt_id];
1502                 rte_eth_stats_get(pt_id, &stats);
1503                 stats.ipackets -= port->stats.ipackets;
1504                 port->stats.ipackets = 0;
1505                 stats.opackets -= port->stats.opackets;
1506                 port->stats.opackets = 0;
1507                 stats.ibytes   -= port->stats.ibytes;
1508                 port->stats.ibytes = 0;
1509                 stats.obytes   -= port->stats.obytes;
1510                 port->stats.obytes = 0;
1511                 stats.imissed  -= port->stats.imissed;
1512                 port->stats.imissed = 0;
1513                 stats.oerrors  -= port->stats.oerrors;
1514                 port->stats.oerrors = 0;
1515                 stats.rx_nombuf -= port->stats.rx_nombuf;
1516                 port->stats.rx_nombuf = 0;
1517
1518                 total_recv += stats.ipackets;
1519                 total_xmit += stats.opackets;
1520                 total_rx_dropped += stats.imissed;
1521                 total_tx_dropped += port->tx_dropped;
1522                 total_rx_nombuf  += stats.rx_nombuf;
1523
1524                 fwd_port_stats_display(pt_id, &stats);
1525         }
1526
1527         printf("\n  %s Accumulated forward statistics for all ports"
1528                "%s\n",
1529                acc_stats_border, acc_stats_border);
1530         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1531                "%-"PRIu64"\n"
1532                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1533                "%-"PRIu64"\n",
1534                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1535                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1536         if (total_rx_nombuf > 0)
1537                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1538         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1539                "%s\n",
1540                acc_stats_border, acc_stats_border);
1541 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1542         if (total_recv > 0)
1543                 printf("\n  CPU cycles/packet=%u (total cycles="
1544                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1545                        (unsigned int)(fwd_cycles / total_recv),
1546                        fwd_cycles, total_recv);
1547 #endif
1548         printf("\nDone.\n");
1549         test_done = 1;
1550 }
1551
1552 void
1553 dev_set_link_up(portid_t pid)
1554 {
1555         if (rte_eth_dev_set_link_up(pid) < 0)
1556                 printf("\nSet link up fail.\n");
1557 }
1558
1559 void
1560 dev_set_link_down(portid_t pid)
1561 {
1562         if (rte_eth_dev_set_link_down(pid) < 0)
1563                 printf("\nSet link down fail.\n");
1564 }
1565
1566 static int
1567 all_ports_started(void)
1568 {
1569         portid_t pi;
1570         struct rte_port *port;
1571
1572         RTE_ETH_FOREACH_DEV(pi) {
1573                 port = &ports[pi];
1574                 /* Check if there is a port which is not started */
1575                 if ((port->port_status != RTE_PORT_STARTED) &&
1576                         (port->slave_flag == 0))
1577                         return 0;
1578         }
1579
1580         /* No port is not started */
1581         return 1;
1582 }
1583
1584 int
1585 port_is_stopped(portid_t port_id)
1586 {
1587         struct rte_port *port = &ports[port_id];
1588
1589         if ((port->port_status != RTE_PORT_STOPPED) &&
1590             (port->slave_flag == 0))
1591                 return 0;
1592         return 1;
1593 }
1594
1595 int
1596 all_ports_stopped(void)
1597 {
1598         portid_t pi;
1599
1600         RTE_ETH_FOREACH_DEV(pi) {
1601                 if (!port_is_stopped(pi))
1602                         return 0;
1603         }
1604
1605         return 1;
1606 }
1607
1608 int
1609 port_is_started(portid_t port_id)
1610 {
1611         if (port_id_is_invalid(port_id, ENABLED_WARN))
1612                 return 0;
1613
1614         if (ports[port_id].port_status != RTE_PORT_STARTED)
1615                 return 0;
1616
1617         return 1;
1618 }
1619
1620 static int
1621 port_is_closed(portid_t port_id)
1622 {
1623         if (port_id_is_invalid(port_id, ENABLED_WARN))
1624                 return 0;
1625
1626         if (ports[port_id].port_status != RTE_PORT_CLOSED)
1627                 return 0;
1628
1629         return 1;
1630 }
1631
1632 int
1633 start_port(portid_t pid)
1634 {
1635         int diag, need_check_link_status = -1;
1636         portid_t pi;
1637         queueid_t qi;
1638         struct rte_port *port;
1639         struct ether_addr mac_addr;
1640         enum rte_eth_event_type event_type;
1641
1642         if (port_id_is_invalid(pid, ENABLED_WARN))
1643                 return 0;
1644
1645         if(dcb_config)
1646                 dcb_test = 1;
1647         RTE_ETH_FOREACH_DEV(pi) {
1648                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1649                         continue;
1650
1651                 need_check_link_status = 0;
1652                 port = &ports[pi];
1653                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1654                                                  RTE_PORT_HANDLING) == 0) {
1655                         printf("Port %d is now not stopped\n", pi);
1656                         continue;
1657                 }
1658
1659                 if (port->need_reconfig > 0) {
1660                         port->need_reconfig = 0;
1661
1662                         if (flow_isolate_all) {
1663                                 int ret = port_flow_isolate(pi, 1);
1664                                 if (ret) {
1665                                         printf("Failed to apply isolated"
1666                                                " mode on port %d\n", pi);
1667                                         return -1;
1668                                 }
1669                         }
1670
1671                         printf("Configuring Port %d (socket %u)\n", pi,
1672                                         port->socket_id);
1673                         /* configure port */
1674                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1675                                                 &(port->dev_conf));
1676                         if (diag != 0) {
1677                                 if (rte_atomic16_cmpset(&(port->port_status),
1678                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1679                                         printf("Port %d can not be set back "
1680                                                         "to stopped\n", pi);
1681                                 printf("Fail to configure port %d\n", pi);
1682                                 /* try to reconfigure port next time */
1683                                 port->need_reconfig = 1;
1684                                 return -1;
1685                         }
1686                 }
1687                 if (port->need_reconfig_queues > 0) {
1688                         port->need_reconfig_queues = 0;
1689                         /* setup tx queues */
1690                         for (qi = 0; qi < nb_txq; qi++) {
1691                                 if ((numa_support) &&
1692                                         (txring_numa[pi] != NUMA_NO_CONFIG))
1693                                         diag = rte_eth_tx_queue_setup(pi, qi,
1694                                                 port->nb_tx_desc[qi],
1695                                                 txring_numa[pi],
1696                                                 &(port->tx_conf[qi]));
1697                                 else
1698                                         diag = rte_eth_tx_queue_setup(pi, qi,
1699                                                 port->nb_tx_desc[qi],
1700                                                 port->socket_id,
1701                                                 &(port->tx_conf[qi]));
1702
1703                                 if (diag == 0)
1704                                         continue;
1705
1706                                 /* Fail to setup tx queue, return */
1707                                 if (rte_atomic16_cmpset(&(port->port_status),
1708                                                         RTE_PORT_HANDLING,
1709                                                         RTE_PORT_STOPPED) == 0)
1710                                         printf("Port %d can not be set back "
1711                                                         "to stopped\n", pi);
1712                                 printf("Fail to configure port %d tx queues\n",
1713                                        pi);
1714                                 /* try to reconfigure queues next time */
1715                                 port->need_reconfig_queues = 1;
1716                                 return -1;
1717                         }
1718                         for (qi = 0; qi < nb_rxq; qi++) {
1719                                 /* setup rx queues */
1720                                 if ((numa_support) &&
1721                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
1722                                         struct rte_mempool * mp =
1723                                                 mbuf_pool_find(rxring_numa[pi]);
1724                                         if (mp == NULL) {
1725                                                 printf("Failed to setup RX queue:"
1726                                                         "No mempool allocation"
1727                                                         " on the socket %d\n",
1728                                                         rxring_numa[pi]);
1729                                                 return -1;
1730                                         }
1731
1732                                         diag = rte_eth_rx_queue_setup(pi, qi,
1733                                              port->nb_rx_desc[qi],
1734                                              rxring_numa[pi],
1735                                              &(port->rx_conf[qi]),
1736                                              mp);
1737                                 } else {
1738                                         struct rte_mempool *mp =
1739                                                 mbuf_pool_find(port->socket_id);
1740                                         if (mp == NULL) {
1741                                                 printf("Failed to setup RX queue:"
1742                                                         "No mempool allocation"
1743                                                         " on the socket %d\n",
1744                                                         port->socket_id);
1745                                                 return -1;
1746                                         }
1747                                         diag = rte_eth_rx_queue_setup(pi, qi,
1748                                              port->nb_rx_desc[qi],
1749                                              port->socket_id,
1750                                              &(port->rx_conf[qi]),
1751                                              mp);
1752                                 }
1753                                 if (diag == 0)
1754                                         continue;
1755
1756                                 /* Fail to setup rx queue, return */
1757                                 if (rte_atomic16_cmpset(&(port->port_status),
1758                                                         RTE_PORT_HANDLING,
1759                                                         RTE_PORT_STOPPED) == 0)
1760                                         printf("Port %d can not be set back "
1761                                                         "to stopped\n", pi);
1762                                 printf("Fail to configure port %d rx queues\n",
1763                                        pi);
1764                                 /* try to reconfigure queues next time */
1765                                 port->need_reconfig_queues = 1;
1766                                 return -1;
1767                         }
1768                 }
1769
1770                 /* start port */
1771                 if (rte_eth_dev_start(pi) < 0) {
1772                         printf("Fail to start port %d\n", pi);
1773
1774                         /* Fail to setup rx queue, return */
1775                         if (rte_atomic16_cmpset(&(port->port_status),
1776                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1777                                 printf("Port %d can not be set back to "
1778                                                         "stopped\n", pi);
1779                         continue;
1780                 }
1781
1782                 if (rte_atomic16_cmpset(&(port->port_status),
1783                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
1784                         printf("Port %d can not be set into started\n", pi);
1785
1786                 rte_eth_macaddr_get(pi, &mac_addr);
1787                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
1788                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
1789                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
1790                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
1791
1792                 /* at least one port started, need checking link status */
1793                 need_check_link_status = 1;
1794         }
1795
1796         for (event_type = RTE_ETH_EVENT_UNKNOWN;
1797              event_type < RTE_ETH_EVENT_MAX;
1798              event_type++) {
1799                 diag = rte_eth_dev_callback_register(RTE_ETH_ALL,
1800                                                 event_type,
1801                                                 eth_event_callback,
1802                                                 NULL);
1803                 if (diag) {
1804                         printf("Failed to setup even callback for event %d\n",
1805                                 event_type);
1806                         return -1;
1807                 }
1808         }
1809
1810         if (need_check_link_status == 1 && !no_link_check)
1811                 check_all_ports_link_status(RTE_PORT_ALL);
1812         else if (need_check_link_status == 0)
1813                 printf("Please stop the ports first\n");
1814
1815         printf("Done\n");
1816         return 0;
1817 }
1818
1819 void
1820 stop_port(portid_t pid)
1821 {
1822         portid_t pi;
1823         struct rte_port *port;
1824         int need_check_link_status = 0;
1825
1826         if (dcb_test) {
1827                 dcb_test = 0;
1828                 dcb_config = 0;
1829         }
1830
1831         if (port_id_is_invalid(pid, ENABLED_WARN))
1832                 return;
1833
1834         printf("Stopping ports...\n");
1835
1836         RTE_ETH_FOREACH_DEV(pi) {
1837                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1838                         continue;
1839
1840                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1841                         printf("Please remove port %d from forwarding configuration.\n", pi);
1842                         continue;
1843                 }
1844
1845                 if (port_is_bonding_slave(pi)) {
1846                         printf("Please remove port %d from bonded device.\n", pi);
1847                         continue;
1848                 }
1849
1850                 port = &ports[pi];
1851                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
1852                                                 RTE_PORT_HANDLING) == 0)
1853                         continue;
1854
1855                 rte_eth_dev_stop(pi);
1856
1857                 if (rte_atomic16_cmpset(&(port->port_status),
1858                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1859                         printf("Port %d can not be set into stopped\n", pi);
1860                 need_check_link_status = 1;
1861         }
1862         if (need_check_link_status && !no_link_check)
1863                 check_all_ports_link_status(RTE_PORT_ALL);
1864
1865         printf("Done\n");
1866 }
1867
1868 void
1869 close_port(portid_t pid)
1870 {
1871         portid_t pi;
1872         struct rte_port *port;
1873
1874         if (port_id_is_invalid(pid, ENABLED_WARN))
1875                 return;
1876
1877         printf("Closing ports...\n");
1878
1879         RTE_ETH_FOREACH_DEV(pi) {
1880                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1881                         continue;
1882
1883                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1884                         printf("Please remove port %d from forwarding configuration.\n", pi);
1885                         continue;
1886                 }
1887
1888                 if (port_is_bonding_slave(pi)) {
1889                         printf("Please remove port %d from bonded device.\n", pi);
1890                         continue;
1891                 }
1892
1893                 port = &ports[pi];
1894                 if (rte_atomic16_cmpset(&(port->port_status),
1895                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
1896                         printf("Port %d is already closed\n", pi);
1897                         continue;
1898                 }
1899
1900                 if (rte_atomic16_cmpset(&(port->port_status),
1901                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
1902                         printf("Port %d is now not stopped\n", pi);
1903                         continue;
1904                 }
1905
1906                 if (port->flow_list)
1907                         port_flow_flush(pi);
1908                 rte_eth_dev_close(pi);
1909
1910                 if (rte_atomic16_cmpset(&(port->port_status),
1911                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
1912                         printf("Port %d cannot be set to closed\n", pi);
1913         }
1914
1915         printf("Done\n");
1916 }
1917
1918 void
1919 reset_port(portid_t pid)
1920 {
1921         int diag;
1922         portid_t pi;
1923         struct rte_port *port;
1924
1925         if (port_id_is_invalid(pid, ENABLED_WARN))
1926                 return;
1927
1928         printf("Resetting ports...\n");
1929
1930         RTE_ETH_FOREACH_DEV(pi) {
1931                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1932                         continue;
1933
1934                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
1935                         printf("Please remove port %d from forwarding "
1936                                "configuration.\n", pi);
1937                         continue;
1938                 }
1939
1940                 if (port_is_bonding_slave(pi)) {
1941                         printf("Please remove port %d from bonded device.\n",
1942                                pi);
1943                         continue;
1944                 }
1945
1946                 diag = rte_eth_dev_reset(pi);
1947                 if (diag == 0) {
1948                         port = &ports[pi];
1949                         port->need_reconfig = 1;
1950                         port->need_reconfig_queues = 1;
1951                 } else {
1952                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
1953                 }
1954         }
1955
1956         printf("Done\n");
1957 }
1958
1959 static int
1960 eth_dev_event_callback_register(void)
1961 {
1962         int ret;
1963
1964         /* register the device event callback */
1965         ret = rte_dev_event_callback_register(NULL,
1966                 eth_dev_event_callback, NULL);
1967         if (ret) {
1968                 printf("Failed to register device event callback\n");
1969                 return -1;
1970         }
1971
1972         return 0;
1973 }
1974
1975
1976 static int
1977 eth_dev_event_callback_unregister(void)
1978 {
1979         int ret;
1980
1981         /* unregister the device event callback */
1982         ret = rte_dev_event_callback_unregister(NULL,
1983                 eth_dev_event_callback, NULL);
1984         if (ret < 0) {
1985                 printf("Failed to unregister device event callback\n");
1986                 return -1;
1987         }
1988
1989         return 0;
1990 }
1991
1992 void
1993 attach_port(char *identifier)
1994 {
1995         portid_t pi = 0;
1996         unsigned int socket_id;
1997
1998         printf("Attaching a new port...\n");
1999
2000         if (identifier == NULL) {
2001                 printf("Invalid parameters are specified\n");
2002                 return;
2003         }
2004
2005         if (rte_eth_dev_attach(identifier, &pi))
2006                 return;
2007
2008         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2009         /* if socket_id is invalid, set to 0 */
2010         if (check_socket_id(socket_id) < 0)
2011                 socket_id = 0;
2012         reconfig(pi, socket_id);
2013         rte_eth_promiscuous_enable(pi);
2014
2015         nb_ports = rte_eth_dev_count_avail();
2016
2017         ports[pi].port_status = RTE_PORT_STOPPED;
2018
2019         update_fwd_ports(pi);
2020
2021         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2022         printf("Done\n");
2023 }
2024
2025 void
2026 detach_port(portid_t port_id)
2027 {
2028         char name[RTE_ETH_NAME_MAX_LEN];
2029
2030         printf("Detaching a port...\n");
2031
2032         if (!port_is_closed(port_id)) {
2033                 printf("Please close port first\n");
2034                 return;
2035         }
2036
2037         if (ports[port_id].flow_list)
2038                 port_flow_flush(port_id);
2039
2040         if (rte_eth_dev_detach(port_id, name)) {
2041                 TESTPMD_LOG(ERR, "Failed to detach port %u\n", port_id);
2042                 return;
2043         }
2044
2045         nb_ports = rte_eth_dev_count_avail();
2046
2047         update_fwd_ports(RTE_MAX_ETHPORTS);
2048
2049         printf("Port %u is detached. Now total ports is %d\n",
2050                         port_id, nb_ports);
2051         printf("Done\n");
2052         return;
2053 }
2054
2055 void
2056 pmd_test_exit(void)
2057 {
2058         struct rte_device *device;
2059         portid_t pt_id;
2060         int ret;
2061
2062         if (test_done == 0)
2063                 stop_packet_forwarding();
2064
2065         if (ports != NULL) {
2066                 no_link_check = 1;
2067                 RTE_ETH_FOREACH_DEV(pt_id) {
2068                         printf("\nShutting down port %d...\n", pt_id);
2069                         fflush(stdout);
2070                         stop_port(pt_id);
2071                         close_port(pt_id);
2072
2073                         /*
2074                          * This is a workaround to fix a virtio-user issue that
2075                          * requires to call clean-up routine to remove existing
2076                          * socket.
2077                          * This workaround valid only for testpmd, needs a fix
2078                          * valid for all applications.
2079                          * TODO: Implement proper resource cleanup
2080                          */
2081                         device = rte_eth_devices[pt_id].device;
2082                         if (device && !strcmp(device->driver->name, "net_virtio_user"))
2083                                 detach_port(pt_id);
2084                 }
2085         }
2086
2087         if (hot_plug) {
2088                 ret = rte_dev_event_monitor_stop();
2089                 if (ret)
2090                         RTE_LOG(ERR, EAL,
2091                                 "fail to stop device event monitor.");
2092
2093                 ret = eth_dev_event_callback_unregister();
2094                 if (ret)
2095                         RTE_LOG(ERR, EAL,
2096                                 "fail to unregister all event callbacks.");
2097         }
2098
2099         printf("\nBye...\n");
2100 }
2101
2102 typedef void (*cmd_func_t)(void);
2103 struct pmd_test_command {
2104         const char *cmd_name;
2105         cmd_func_t cmd_func;
2106 };
2107
2108 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2109
2110 /* Check the link status of all ports in up to 9s, and print them finally */
2111 static void
2112 check_all_ports_link_status(uint32_t port_mask)
2113 {
2114 #define CHECK_INTERVAL 100 /* 100ms */
2115 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2116         portid_t portid;
2117         uint8_t count, all_ports_up, print_flag = 0;
2118         struct rte_eth_link link;
2119
2120         printf("Checking link statuses...\n");
2121         fflush(stdout);
2122         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2123                 all_ports_up = 1;
2124                 RTE_ETH_FOREACH_DEV(portid) {
2125                         if ((port_mask & (1 << portid)) == 0)
2126                                 continue;
2127                         memset(&link, 0, sizeof(link));
2128                         rte_eth_link_get_nowait(portid, &link);
2129                         /* print link status if flag set */
2130                         if (print_flag == 1) {
2131                                 if (link.link_status)
2132                                         printf(
2133                                         "Port%d Link Up. speed %u Mbps- %s\n",
2134                                         portid, link.link_speed,
2135                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2136                                         ("full-duplex") : ("half-duplex\n"));
2137                                 else
2138                                         printf("Port %d Link Down\n", portid);
2139                                 continue;
2140                         }
2141                         /* clear all_ports_up flag if any link down */
2142                         if (link.link_status == ETH_LINK_DOWN) {
2143                                 all_ports_up = 0;
2144                                 break;
2145                         }
2146                 }
2147                 /* after finally printing all link status, get out */
2148                 if (print_flag == 1)
2149                         break;
2150
2151                 if (all_ports_up == 0) {
2152                         fflush(stdout);
2153                         rte_delay_ms(CHECK_INTERVAL);
2154                 }
2155
2156                 /* set the print_flag if all ports up or timeout */
2157                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2158                         print_flag = 1;
2159                 }
2160
2161                 if (lsc_interrupt)
2162                         break;
2163         }
2164 }
2165
2166 static void
2167 rmv_event_callback(void *arg)
2168 {
2169         int need_to_start = 0;
2170         int org_no_link_check = no_link_check;
2171         portid_t port_id = (intptr_t)arg;
2172
2173         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2174
2175         if (!test_done && port_is_forwarding(port_id)) {
2176                 need_to_start = 1;
2177                 stop_packet_forwarding();
2178         }
2179         no_link_check = 1;
2180         stop_port(port_id);
2181         no_link_check = org_no_link_check;
2182         close_port(port_id);
2183         detach_port(port_id);
2184         if (need_to_start)
2185                 start_packet_forwarding(0);
2186 }
2187
2188 /* This function is used by the interrupt thread */
2189 static int
2190 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2191                   void *ret_param)
2192 {
2193         static const char * const event_desc[] = {
2194                 [RTE_ETH_EVENT_UNKNOWN] = "Unknown",
2195                 [RTE_ETH_EVENT_INTR_LSC] = "LSC",
2196                 [RTE_ETH_EVENT_QUEUE_STATE] = "Queue state",
2197                 [RTE_ETH_EVENT_INTR_RESET] = "Interrupt reset",
2198                 [RTE_ETH_EVENT_VF_MBOX] = "VF Mbox",
2199                 [RTE_ETH_EVENT_IPSEC] = "IPsec",
2200                 [RTE_ETH_EVENT_MACSEC] = "MACsec",
2201                 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
2202                 [RTE_ETH_EVENT_NEW] = "device probed",
2203                 [RTE_ETH_EVENT_DESTROY] = "device released",
2204                 [RTE_ETH_EVENT_MAX] = NULL,
2205         };
2206
2207         RTE_SET_USED(param);
2208         RTE_SET_USED(ret_param);
2209
2210         if (type >= RTE_ETH_EVENT_MAX) {
2211                 fprintf(stderr, "\nPort %" PRIu8 ": %s called upon invalid event %d\n",
2212                         port_id, __func__, type);
2213                 fflush(stderr);
2214         } else if (event_print_mask & (UINT32_C(1) << type)) {
2215                 printf("\nPort %" PRIu8 ": %s event\n", port_id,
2216                         event_desc[type]);
2217                 fflush(stdout);
2218         }
2219
2220         if (port_id_is_invalid(port_id, DISABLED_WARN))
2221                 return 0;
2222
2223         switch (type) {
2224         case RTE_ETH_EVENT_INTR_RMV:
2225                 if (rte_eal_alarm_set(100000,
2226                                 rmv_event_callback, (void *)(intptr_t)port_id))
2227                         fprintf(stderr, "Could not set up deferred device removal\n");
2228                 break;
2229         default:
2230                 break;
2231         }
2232         return 0;
2233 }
2234
2235 /* This function is used by the interrupt thread */
2236 static void
2237 eth_dev_event_callback(char *device_name, enum rte_dev_event_type type,
2238                              __rte_unused void *arg)
2239 {
2240         if (type >= RTE_DEV_EVENT_MAX) {
2241                 fprintf(stderr, "%s called upon invalid event %d\n",
2242                         __func__, type);
2243                 fflush(stderr);
2244         }
2245
2246         switch (type) {
2247         case RTE_DEV_EVENT_REMOVE:
2248                 RTE_LOG(ERR, EAL, "The device: %s has been removed!\n",
2249                         device_name);
2250                 /* TODO: After finish failure handle, begin to stop
2251                  * packet forward, stop port, close port, detach port.
2252                  */
2253                 break;
2254         case RTE_DEV_EVENT_ADD:
2255                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2256                         device_name);
2257                 /* TODO: After finish kernel driver binding,
2258                  * begin to attach port.
2259                  */
2260                 break;
2261         default:
2262                 break;
2263         }
2264 }
2265
2266 static int
2267 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2268 {
2269         uint16_t i;
2270         int diag;
2271         uint8_t mapping_found = 0;
2272
2273         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2274                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2275                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2276                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2277                                         tx_queue_stats_mappings[i].queue_id,
2278                                         tx_queue_stats_mappings[i].stats_counter_id);
2279                         if (diag != 0)
2280                                 return diag;
2281                         mapping_found = 1;
2282                 }
2283         }
2284         if (mapping_found)
2285                 port->tx_queue_stats_mapping_enabled = 1;
2286         return 0;
2287 }
2288
2289 static int
2290 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2291 {
2292         uint16_t i;
2293         int diag;
2294         uint8_t mapping_found = 0;
2295
2296         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2297                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2298                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2299                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2300                                         rx_queue_stats_mappings[i].queue_id,
2301                                         rx_queue_stats_mappings[i].stats_counter_id);
2302                         if (diag != 0)
2303                                 return diag;
2304                         mapping_found = 1;
2305                 }
2306         }
2307         if (mapping_found)
2308                 port->rx_queue_stats_mapping_enabled = 1;
2309         return 0;
2310 }
2311
2312 static void
2313 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2314 {
2315         int diag = 0;
2316
2317         diag = set_tx_queue_stats_mapping_registers(pi, port);
2318         if (diag != 0) {
2319                 if (diag == -ENOTSUP) {
2320                         port->tx_queue_stats_mapping_enabled = 0;
2321                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2322                 }
2323                 else
2324                         rte_exit(EXIT_FAILURE,
2325                                         "set_tx_queue_stats_mapping_registers "
2326                                         "failed for port id=%d diag=%d\n",
2327                                         pi, diag);
2328         }
2329
2330         diag = set_rx_queue_stats_mapping_registers(pi, port);
2331         if (diag != 0) {
2332                 if (diag == -ENOTSUP) {
2333                         port->rx_queue_stats_mapping_enabled = 0;
2334                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2335                 }
2336                 else
2337                         rte_exit(EXIT_FAILURE,
2338                                         "set_rx_queue_stats_mapping_registers "
2339                                         "failed for port id=%d diag=%d\n",
2340                                         pi, diag);
2341         }
2342 }
2343
2344 static void
2345 rxtx_port_config(struct rte_port *port)
2346 {
2347         uint16_t qid;
2348
2349         for (qid = 0; qid < nb_rxq; qid++) {
2350                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2351
2352                 /* Check if any Rx parameters have been passed */
2353                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2354                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2355
2356                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2357                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2358
2359                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2360                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2361
2362                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2363                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2364
2365                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2366                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2367
2368                 port->nb_rx_desc[qid] = nb_rxd;
2369         }
2370
2371         for (qid = 0; qid < nb_txq; qid++) {
2372                 port->tx_conf[qid] = port->dev_info.default_txconf;
2373
2374                 /* Check if any Tx parameters have been passed */
2375                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2376                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2377
2378                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2379                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2380
2381                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2382                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2383
2384                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2385                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2386
2387                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2388                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2389
2390                 port->nb_tx_desc[qid] = nb_txd;
2391         }
2392 }
2393
2394 void
2395 init_port_config(void)
2396 {
2397         portid_t pid;
2398         struct rte_port *port;
2399
2400         RTE_ETH_FOREACH_DEV(pid) {
2401                 port = &ports[pid];
2402                 port->dev_conf.fdir_conf = fdir_conf;
2403                 rte_eth_dev_info_get(pid, &port->dev_info);
2404                 if (nb_rxq > 1) {
2405                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2406                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2407                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2408                 } else {
2409                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2410                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2411                 }
2412
2413                 if (port->dcb_flag == 0) {
2414                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2415                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2416                         else
2417                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2418                 }
2419
2420                 rxtx_port_config(port);
2421
2422                 rte_eth_macaddr_get(pid, &port->eth_addr);
2423
2424                 map_port_queue_stats_mapping_registers(pid, port);
2425 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2426                 rte_pmd_ixgbe_bypass_init(pid);
2427 #endif
2428
2429                 if (lsc_interrupt &&
2430                     (rte_eth_devices[pid].data->dev_flags &
2431                      RTE_ETH_DEV_INTR_LSC))
2432                         port->dev_conf.intr_conf.lsc = 1;
2433                 if (rmv_interrupt &&
2434                     (rte_eth_devices[pid].data->dev_flags &
2435                      RTE_ETH_DEV_INTR_RMV))
2436                         port->dev_conf.intr_conf.rmv = 1;
2437         }
2438 }
2439
2440 void set_port_slave_flag(portid_t slave_pid)
2441 {
2442         struct rte_port *port;
2443
2444         port = &ports[slave_pid];
2445         port->slave_flag = 1;
2446 }
2447
2448 void clear_port_slave_flag(portid_t slave_pid)
2449 {
2450         struct rte_port *port;
2451
2452         port = &ports[slave_pid];
2453         port->slave_flag = 0;
2454 }
2455
2456 uint8_t port_is_bonding_slave(portid_t slave_pid)
2457 {
2458         struct rte_port *port;
2459
2460         port = &ports[slave_pid];
2461         if ((rte_eth_devices[slave_pid].data->dev_flags &
2462             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
2463                 return 1;
2464         return 0;
2465 }
2466
2467 const uint16_t vlan_tags[] = {
2468                 0,  1,  2,  3,  4,  5,  6,  7,
2469                 8,  9, 10, 11,  12, 13, 14, 15,
2470                 16, 17, 18, 19, 20, 21, 22, 23,
2471                 24, 25, 26, 27, 28, 29, 30, 31
2472 };
2473
2474 static  int
2475 get_eth_dcb_conf(struct rte_eth_conf *eth_conf,
2476                  enum dcb_mode_enable dcb_mode,
2477                  enum rte_eth_nb_tcs num_tcs,
2478                  uint8_t pfc_en)
2479 {
2480         uint8_t i;
2481
2482         /*
2483          * Builds up the correct configuration for dcb+vt based on the vlan tags array
2484          * given above, and the number of traffic classes available for use.
2485          */
2486         if (dcb_mode == DCB_VT_ENABLED) {
2487                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
2488                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
2489                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2490                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
2491
2492                 /* VMDQ+DCB RX and TX configurations */
2493                 vmdq_rx_conf->enable_default_pool = 0;
2494                 vmdq_rx_conf->default_pool = 0;
2495                 vmdq_rx_conf->nb_queue_pools =
2496                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2497                 vmdq_tx_conf->nb_queue_pools =
2498                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2499
2500                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
2501                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
2502                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
2503                         vmdq_rx_conf->pool_map[i].pools =
2504                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
2505                 }
2506                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2507                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
2508                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
2509                 }
2510
2511                 /* set DCB mode of RX and TX of multiple queues */
2512                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
2513                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
2514         } else {
2515                 struct rte_eth_dcb_rx_conf *rx_conf =
2516                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
2517                 struct rte_eth_dcb_tx_conf *tx_conf =
2518                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
2519
2520                 rx_conf->nb_tcs = num_tcs;
2521                 tx_conf->nb_tcs = num_tcs;
2522
2523                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2524                         rx_conf->dcb_tc[i] = i % num_tcs;
2525                         tx_conf->dcb_tc[i] = i % num_tcs;
2526                 }
2527                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
2528                 eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf;
2529                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
2530         }
2531
2532         if (pfc_en)
2533                 eth_conf->dcb_capability_en =
2534                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
2535         else
2536                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
2537
2538         return 0;
2539 }
2540
2541 int
2542 init_port_dcb_config(portid_t pid,
2543                      enum dcb_mode_enable dcb_mode,
2544                      enum rte_eth_nb_tcs num_tcs,
2545                      uint8_t pfc_en)
2546 {
2547         struct rte_eth_conf port_conf;
2548         struct rte_port *rte_port;
2549         int retval;
2550         uint16_t i;
2551
2552         rte_port = &ports[pid];
2553
2554         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
2555         /* Enter DCB configuration status */
2556         dcb_config = 1;
2557
2558         port_conf.rxmode = rte_port->dev_conf.rxmode;
2559         port_conf.txmode = rte_port->dev_conf.txmode;
2560
2561         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
2562         retval = get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en);
2563         if (retval < 0)
2564                 return retval;
2565         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
2566
2567         /* re-configure the device . */
2568         rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
2569
2570         rte_eth_dev_info_get(pid, &rte_port->dev_info);
2571
2572         /* If dev_info.vmdq_pool_base is greater than 0,
2573          * the queue id of vmdq pools is started after pf queues.
2574          */
2575         if (dcb_mode == DCB_VT_ENABLED &&
2576             rte_port->dev_info.vmdq_pool_base > 0) {
2577                 printf("VMDQ_DCB multi-queue mode is nonsensical"
2578                         " for port %d.", pid);
2579                 return -1;
2580         }
2581
2582         /* Assume the ports in testpmd have the same dcb capability
2583          * and has the same number of rxq and txq in dcb mode
2584          */
2585         if (dcb_mode == DCB_VT_ENABLED) {
2586                 if (rte_port->dev_info.max_vfs > 0) {
2587                         nb_rxq = rte_port->dev_info.nb_rx_queues;
2588                         nb_txq = rte_port->dev_info.nb_tx_queues;
2589                 } else {
2590                         nb_rxq = rte_port->dev_info.max_rx_queues;
2591                         nb_txq = rte_port->dev_info.max_tx_queues;
2592                 }
2593         } else {
2594                 /*if vt is disabled, use all pf queues */
2595                 if (rte_port->dev_info.vmdq_pool_base == 0) {
2596                         nb_rxq = rte_port->dev_info.max_rx_queues;
2597                         nb_txq = rte_port->dev_info.max_tx_queues;
2598                 } else {
2599                         nb_rxq = (queueid_t)num_tcs;
2600                         nb_txq = (queueid_t)num_tcs;
2601
2602                 }
2603         }
2604         rx_free_thresh = 64;
2605
2606         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
2607
2608         rxtx_port_config(rte_port);
2609         /* VLAN filter */
2610         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
2611         for (i = 0; i < RTE_DIM(vlan_tags); i++)
2612                 rx_vft_set(pid, vlan_tags[i], 1);
2613
2614         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
2615         map_port_queue_stats_mapping_registers(pid, rte_port);
2616
2617         rte_port->dcb_flag = 1;
2618
2619         return 0;
2620 }
2621
2622 static void
2623 init_port(void)
2624 {
2625         /* Configuration of Ethernet ports. */
2626         ports = rte_zmalloc("testpmd: ports",
2627                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
2628                             RTE_CACHE_LINE_SIZE);
2629         if (ports == NULL) {
2630                 rte_exit(EXIT_FAILURE,
2631                                 "rte_zmalloc(%d struct rte_port) failed\n",
2632                                 RTE_MAX_ETHPORTS);
2633         }
2634 }
2635
2636 static void
2637 force_quit(void)
2638 {
2639         pmd_test_exit();
2640         prompt_exit();
2641 }
2642
2643 static void
2644 print_stats(void)
2645 {
2646         uint8_t i;
2647         const char clr[] = { 27, '[', '2', 'J', '\0' };
2648         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
2649
2650         /* Clear screen and move to top left */
2651         printf("%s%s", clr, top_left);
2652
2653         printf("\nPort statistics ====================================");
2654         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2655                 nic_stats_display(fwd_ports_ids[i]);
2656 }
2657
2658 static void
2659 signal_handler(int signum)
2660 {
2661         if (signum == SIGINT || signum == SIGTERM) {
2662                 printf("\nSignal %d received, preparing to exit...\n",
2663                                 signum);
2664 #ifdef RTE_LIBRTE_PDUMP
2665                 /* uninitialize packet capture framework */
2666                 rte_pdump_uninit();
2667 #endif
2668 #ifdef RTE_LIBRTE_LATENCY_STATS
2669                 rte_latencystats_uninit();
2670 #endif
2671                 force_quit();
2672                 /* Set flag to indicate the force termination. */
2673                 f_quit = 1;
2674                 /* exit with the expected status */
2675                 signal(signum, SIG_DFL);
2676                 kill(getpid(), signum);
2677         }
2678 }
2679
2680 int
2681 main(int argc, char** argv)
2682 {
2683         int diag;
2684         portid_t port_id;
2685         int ret;
2686
2687         signal(SIGINT, signal_handler);
2688         signal(SIGTERM, signal_handler);
2689
2690         diag = rte_eal_init(argc, argv);
2691         if (diag < 0)
2692                 rte_panic("Cannot init EAL\n");
2693
2694         testpmd_logtype = rte_log_register("testpmd");
2695         if (testpmd_logtype < 0)
2696                 rte_panic("Cannot register log type");
2697         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
2698
2699 #ifdef RTE_LIBRTE_PDUMP
2700         /* initialize packet capture framework */
2701         rte_pdump_init(NULL);
2702 #endif
2703
2704         nb_ports = (portid_t) rte_eth_dev_count_avail();
2705         if (nb_ports == 0)
2706                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
2707
2708         /* allocate port structures, and init them */
2709         init_port();
2710
2711         set_def_fwd_config();
2712         if (nb_lcores == 0)
2713                 rte_panic("Empty set of forwarding logical cores - check the "
2714                           "core mask supplied in the command parameters\n");
2715
2716         /* Bitrate/latency stats disabled by default */
2717 #ifdef RTE_LIBRTE_BITRATE
2718         bitrate_enabled = 0;
2719 #endif
2720 #ifdef RTE_LIBRTE_LATENCY_STATS
2721         latencystats_enabled = 0;
2722 #endif
2723
2724         /* on FreeBSD, mlockall() is disabled by default */
2725 #ifdef RTE_EXEC_ENV_BSDAPP
2726         do_mlockall = 0;
2727 #else
2728         do_mlockall = 1;
2729 #endif
2730
2731         argc -= diag;
2732         argv += diag;
2733         if (argc > 1)
2734                 launch_args_parse(argc, argv);
2735
2736         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
2737                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
2738                         strerror(errno));
2739         }
2740
2741         if (tx_first && interactive)
2742                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
2743                                 "interactive mode.\n");
2744
2745         if (tx_first && lsc_interrupt) {
2746                 printf("Warning: lsc_interrupt needs to be off when "
2747                                 " using tx_first. Disabling.\n");
2748                 lsc_interrupt = 0;
2749         }
2750
2751         if (!nb_rxq && !nb_txq)
2752                 printf("Warning: Either rx or tx queues should be non-zero\n");
2753
2754         if (nb_rxq > 1 && nb_rxq > nb_txq)
2755                 printf("Warning: nb_rxq=%d enables RSS configuration, "
2756                        "but nb_txq=%d will prevent to fully test it.\n",
2757                        nb_rxq, nb_txq);
2758
2759         init_config();
2760
2761         if (hot_plug) {
2762                 /* enable hot plug monitoring */
2763                 ret = rte_dev_event_monitor_start();
2764                 if (ret) {
2765                         rte_errno = EINVAL;
2766                         return -1;
2767                 }
2768                 eth_dev_event_callback_register();
2769
2770         }
2771
2772         if (start_port(RTE_PORT_ALL) != 0)
2773                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
2774
2775         /* set all ports to promiscuous mode by default */
2776         RTE_ETH_FOREACH_DEV(port_id)
2777                 rte_eth_promiscuous_enable(port_id);
2778
2779         /* Init metrics library */
2780         rte_metrics_init(rte_socket_id());
2781
2782 #ifdef RTE_LIBRTE_LATENCY_STATS
2783         if (latencystats_enabled != 0) {
2784                 int ret = rte_latencystats_init(1, NULL);
2785                 if (ret)
2786                         printf("Warning: latencystats init()"
2787                                 " returned error %d\n", ret);
2788                 printf("Latencystats running on lcore %d\n",
2789                         latencystats_lcore_id);
2790         }
2791 #endif
2792
2793         /* Setup bitrate stats */
2794 #ifdef RTE_LIBRTE_BITRATE
2795         if (bitrate_enabled != 0) {
2796                 bitrate_data = rte_stats_bitrate_create();
2797                 if (bitrate_data == NULL)
2798                         rte_exit(EXIT_FAILURE,
2799                                 "Could not allocate bitrate data.\n");
2800                 rte_stats_bitrate_reg(bitrate_data);
2801         }
2802 #endif
2803
2804 #ifdef RTE_LIBRTE_CMDLINE
2805         if (strlen(cmdline_filename) != 0)
2806                 cmdline_read_from_file(cmdline_filename);
2807
2808         if (interactive == 1) {
2809                 if (auto_start) {
2810                         printf("Start automatic packet forwarding\n");
2811                         start_packet_forwarding(0);
2812                 }
2813                 prompt();
2814                 pmd_test_exit();
2815         } else
2816 #endif
2817         {
2818                 char c;
2819                 int rc;
2820
2821                 f_quit = 0;
2822
2823                 printf("No commandline core given, start packet forwarding\n");
2824                 start_packet_forwarding(tx_first);
2825                 if (stats_period != 0) {
2826                         uint64_t prev_time = 0, cur_time, diff_time = 0;
2827                         uint64_t timer_period;
2828
2829                         /* Convert to number of cycles */
2830                         timer_period = stats_period * rte_get_timer_hz();
2831
2832                         while (f_quit == 0) {
2833                                 cur_time = rte_get_timer_cycles();
2834                                 diff_time += cur_time - prev_time;
2835
2836                                 if (diff_time >= timer_period) {
2837                                         print_stats();
2838                                         /* Reset the timer */
2839                                         diff_time = 0;
2840                                 }
2841                                 /* Sleep to avoid unnecessary checks */
2842                                 prev_time = cur_time;
2843                                 sleep(1);
2844                         }
2845                 }
2846
2847                 printf("Press enter to exit\n");
2848                 rc = read(0, &c, 1);
2849                 pmd_test_exit();
2850                 if (rc < 0)
2851                         return 1;
2852         }
2853
2854         return 0;
2855 }