app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATE
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81
  82 uint16_t verbose_level = 0; /**< Silent by default. */
  83 int testpmd_logtype; /**< Log type for testpmd logs */
  84
  85 /* use master core for command line ? */
  86 uint8_t interactive = 0;
  87 uint8_t auto_start = 0;
  88 uint8_t tx_first;
  89 char cmdline_filename[PATH_MAX] = {0};
  90
  91 /*
  92  * NUMA support configuration.
  93  * When set, the NUMA support attempts to dispatch the allocation of the
  94  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  95  * probed ports among the CPU sockets 0 and 1.
  96  * Otherwise, all memory is allocated from CPU socket 0.
  97  */
  98 uint8_t numa_support = 1; /**< numa enabled by default */
  99
 100 /*
 101  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 102  * not configured.
 103  */
 104 uint8_t socket_num = UMA_NO_CONFIG;
 105
 106 /*
 107  * Select mempool allocation type:
 108  * - native: use regular DPDK memory
 109  * - anon: use regular DPDK memory to create mempool, but populate using
 110  *         anonymous memory (may not be IOVA-contiguous)
 111  * - xmem: use externally allocated hugepage memory
 112  */
 113 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 114
 115 /*
 116  * Store specified sockets on which memory pool to be used by ports
 117  * is allocated.
 118  */
 119 uint8_t port_numa[RTE_MAX_ETHPORTS];
 120
 121 /*
 122  * Store specified sockets on which RX ring to be used by ports
 123  * is allocated.
 124  */
 125 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 126
 127 /*
 128  * Store specified sockets on which TX ring to be used by ports
 129  * is allocated.
 130  */
 131 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 132
 133 /*
 134  * Record the Ethernet address of peer target ports to which packets are
 135  * forwarded.
 136  * Must be instantiated with the ethernet addresses of peer traffic generator
 137  * ports.
 138  */
 139 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 140 portid_t nb_peer_eth_addrs = 0;
 141
 142 /*
 143  * Probed Target Environment.
 144  */
 145 struct rte_port *ports;        /**< For all probed ethernet ports. */
 146 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 147 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 148 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 149
 150 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 151
 152 /*
 153  * Test Forwarding Configuration.
 154  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 155  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 156  */
 157 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 158 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 159 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 160 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 161
 162 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 163 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 164
 165 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 166 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 167
 168 /*
 169  * Forwarding engines.
 170  */
 171 struct fwd_engine * fwd_engines[] = {
 172         &io_fwd_engine,
 173         &mac_fwd_engine,
 174         &mac_swap_engine,
 175         &flow_gen_engine,
 176         &rx_only_engine,
 177         &tx_only_engine,
 178         &csum_fwd_engine,
 179         &icmp_echo_engine,
 180         &noisy_vnf_engine,
 181 #if defined RTE_LIBRTE_PMD_SOFTNIC
 182         &softnic_fwd_engine,
 183 #endif
 184 #ifdef RTE_LIBRTE_IEEE1588
 185         &ieee1588_fwd_engine,
 186 #endif
 187         NULL,
 188 };
 189
 190 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 191 uint16_t mempool_flags;
 192
 193 struct fwd_config cur_fwd_config;
 194 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 195 uint32_t retry_enabled;
 196 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 197 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 198
 199 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 200 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 201                                       * specified on command-line. */
 202 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 203
 204 /*
 205  * In container, it cannot terminate the process which running with 'stats-period'
 206  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 207  */
 208 uint8_t f_quit;
 209
 210 /*
 211  * Configuration of packet segments used by the "txonly" processing engine.
 212  */
 213 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 214 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 215         TXONLY_DEF_PACKET_LEN,
 216 };
 217 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 218
 219 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 220 /**< Split policy for packets to TX. */
 221
 222 uint8_t txonly_multi_flow;
 223 /**< Whether multiple flows are generated in TXONLY mode. */
 224
 225 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 226 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 227
 228 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 229 uint8_t dcb_config = 0;
 230
 231 /* Whether the dcb is in testing status */
 232 uint8_t dcb_test = 0;
 233
 234 /*
 235  * Configurable number of RX/TX queues.
 236  */
 237 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 240
 241 /*
 242  * Configurable number of RX/TX ring descriptors.
 243  * Defaults are supplied by drivers via ethdev.
 244  */
 245 #define RTE_TEST_RX_DESC_DEFAULT 0
 246 #define RTE_TEST_TX_DESC_DEFAULT 0
 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 249
 250 #define RTE_PMD_PARAM_UNSET -1
 251 /*
 252  * Configurable values of RX and TX ring threshold registers.
 253  */
 254
 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 258
 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 262
 263 /*
 264  * Configurable value of RX free threshold.
 265  */
 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 267
 268 /*
 269  * Configurable value of RX drop enable.
 270  */
 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 272
 273 /*
 274  * Configurable value of TX free threshold.
 275  */
 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of TX RS bit threshold.
 280  */
 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of buffered packets before sending.
 285  */
 286 uint16_t noisy_tx_sw_bufsz;
 287
 288 /*
 289  * Configurable value of packet buffer timeout.
 290  */
 291 uint16_t noisy_tx_sw_buf_flush_time;
 292
 293 /*
 294  * Configurable value for size of VNF internal memory area
 295  * used for simulating noisy neighbour behaviour
 296  */
 297 uint64_t noisy_lkup_mem_sz;
 298
 299 /*
 300  * Configurable value of number of random writes done in
 301  * VNF simulation memory area.
 302  */
 303 uint64_t noisy_lkup_num_writes;
 304
 305 /*
 306  * Configurable value of number of random reads done in
 307  * VNF simulation memory area.
 308  */
 309 uint64_t noisy_lkup_num_reads;
 310
 311 /*
 312  * Configurable value of number of random reads/writes done in
 313  * VNF simulation memory area.
 314  */
 315 uint64_t noisy_lkup_num_reads_writes;
 316
 317 /*
 318  * Receive Side Scaling (RSS) configuration.
 319  */
 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 321
 322 /*
 323  * Port topology configuration
 324  */
 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 326
 327 /*
 328  * Avoids to flush all the RX streams before starts forwarding.
 329  */
 330 uint8_t no_flush_rx = 0; /* flush by default */
 331
 332 /*
 333  * Flow API isolated mode.
 334  */
 335 uint8_t flow_isolate_all;
 336
 337 /*
 338  * Avoids to check link status when starting/stopping a port.
 339  */
 340 uint8_t no_link_check = 0; /* check by default */
 341
 342 /*
 343  * Don't automatically start all ports in interactive mode.
 344  */
 345 uint8_t no_device_start = 0;
 346
 347 /*
 348  * Enable link status change notification
 349  */
 350 uint8_t lsc_interrupt = 1; /* enabled by default */
 351
 352 /*
 353  * Enable device removal notification.
 354  */
 355 uint8_t rmv_interrupt = 1; /* enabled by default */
 356
 357 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 358
 359 /* After attach, port setup is called on event or by iterator */
 360 bool setup_on_probe_event = true;
 361
 362 /* Pretty printing of ethdev events */
 363 static const char * const eth_event_desc[] = {
 364         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 365         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 366         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 367         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 368         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 369         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 370         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 371         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 372         [RTE_ETH_EVENT_NEW] = "device probed",
 373         [RTE_ETH_EVENT_DESTROY] = "device released",
 374         [RTE_ETH_EVENT_MAX] = NULL,
 375 };
 376
 377 /*
 378  * Display or mask ether events
 379  * Default to all events except VF_MBOX
 380  */
 381 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 382                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 383                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 384                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 385                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 386                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 387                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 388 /*
 389  * Decide if all memory are locked for performance.
 390  */
 391 int do_mlockall = 0;
 392
 393 /*
 394  * NIC bypass mode configuration options.
 395  */
 396
 397 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 398 /* The NIC bypass watchdog timeout. */
 399 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 400 #endif
 401
 402
 403 #ifdef RTE_LIBRTE_LATENCY_STATS
 404
 405 /*
 406  * Set when latency stats is enabled in the commandline
 407  */
 408 uint8_t latencystats_enabled;
 409
 410 /*
 411  * Lcore ID to serive latency statistics.
 412  */
 413 lcoreid_t latencystats_lcore_id = -1;
 414
 415 #endif
 416
 417 /*
 418  * Ethernet device configuration.
 419  */
 420 struct rte_eth_rxmode rx_mode = {
 421         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 422                 /**< Default maximum frame length. */
 423 };
 424
 425 struct rte_eth_txmode tx_mode = {
 426         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 427 };
 428
 429 struct rte_fdir_conf fdir_conf = {
 430         .mode = RTE_FDIR_MODE_NONE,
 431         .pballoc = RTE_FDIR_PBALLOC_64K,
 432         .status = RTE_FDIR_REPORT_STATUS,
 433         .mask = {
 434                 .vlan_tci_mask = 0xFFEF,
 435                 .ipv4_mask     = {
 436                         .src_ip = 0xFFFFFFFF,
 437                         .dst_ip = 0xFFFFFFFF,
 438                 },
 439                 .ipv6_mask     = {
 440                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 441                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 442                 },
 443                 .src_port_mask = 0xFFFF,
 444                 .dst_port_mask = 0xFFFF,
 445                 .mac_addr_byte_mask = 0xFF,
 446                 .tunnel_type_mask = 1,
 447                 .tunnel_id_mask = 0xFFFFFFFF,
 448         },
 449         .drop_queue = 127,
 450 };
 451
 452 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 453
 454 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 455 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 456
 457 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 458 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 459
 460 uint16_t nb_tx_queue_stats_mappings = 0;
 461 uint16_t nb_rx_queue_stats_mappings = 0;
 462
 463 /*
 464  * Display zero values by default for xstats
 465  */
 466 uint8_t xstats_hide_zero;
 467
 468 unsigned int num_sockets = 0;
 469 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 470
 471 #ifdef RTE_LIBRTE_BITRATE
 472 /* Bitrate statistics */
 473 struct rte_stats_bitrates *bitrate_data;
 474 lcoreid_t bitrate_lcore_id;
 475 uint8_t bitrate_enabled;
 476 #endif
 477
 478 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 479 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 480
 481 /* Forward function declarations */
 482 static void setup_attached_port(portid_t pi);
 483 static void map_port_queue_stats_mapping_registers(portid_t pi,
 484                                                    struct rte_port *port);
 485 static void check_all_ports_link_status(uint32_t port_mask);
 486 static int eth_event_callback(portid_t port_id,
 487                               enum rte_eth_event_type type,
 488                               void *param, void *ret_param);
 489 static void dev_event_callback(const char *device_name,
 490                                 enum rte_dev_event_type type,
 491                                 void *param);
 492
 493 /*
 494  * Check if all the ports are started.
 495  * If yes, return positive value. If not, return zero.
 496  */
 497 static int all_ports_started(void);
 498
 499 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 500 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 501
 502 /*
 503  * Helper function to check if socket is already discovered.
 504  * If yes, return positive value. If not, return zero.
 505  */
 506 int
 507 new_socket_id(unsigned int socket_id)
 508 {
 509         unsigned int i;
 510
 511         for (i = 0; i < num_sockets; i++) {
 512                 if (socket_ids[i] == socket_id)
 513                         return 0;
 514         }
 515         return 1;
 516 }
 517
 518 /*
 519  * Setup default configuration.
 520  */
 521 static void
 522 set_default_fwd_lcores_config(void)
 523 {
 524         unsigned int i;
 525         unsigned int nb_lc;
 526         unsigned int sock_num;
 527
 528         nb_lc = 0;
 529         for (i = 0; i < RTE_MAX_LCORE; i++) {
 530                 if (!rte_lcore_is_enabled(i))
 531                         continue;
 532                 sock_num = rte_lcore_to_socket_id(i);
 533                 if (new_socket_id(sock_num)) {
 534                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 535                                 rte_exit(EXIT_FAILURE,
 536                                          "Total sockets greater than %u\n",
 537                                          RTE_MAX_NUMA_NODES);
 538                         }
 539                         socket_ids[num_sockets++] = sock_num;
 540                 }
 541                 if (i == rte_get_master_lcore())
 542                         continue;
 543                 fwd_lcores_cpuids[nb_lc++] = i;
 544         }
 545         nb_lcores = (lcoreid_t) nb_lc;
 546         nb_cfg_lcores = nb_lcores;
 547         nb_fwd_lcores = 1;
 548 }
 549
 550 static void
 551 set_def_peer_eth_addrs(void)
 552 {
 553         portid_t i;
 554
 555         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 556                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 557                 peer_eth_addrs[i].addr_bytes[5] = i;
 558         }
 559 }
 560
 561 static void
 562 set_default_fwd_ports_config(void)
 563 {
 564         portid_t pt_id;
 565         int i = 0;
 566
 567         RTE_ETH_FOREACH_DEV(pt_id) {
 568                 fwd_ports_ids[i++] = pt_id;
 569
 570                 /* Update sockets info according to the attached device */
 571                 int socket_id = rte_eth_dev_socket_id(pt_id);
 572                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 573                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 574                                 rte_exit(EXIT_FAILURE,
 575                                          "Total sockets greater than %u\n",
 576                                          RTE_MAX_NUMA_NODES);
 577                         }
 578                         socket_ids[num_sockets++] = socket_id;
 579                 }
 580         }
 581
 582         nb_cfg_ports = nb_ports;
 583         nb_fwd_ports = nb_ports;
 584 }
 585
 586 void
 587 set_def_fwd_config(void)
 588 {
 589         set_default_fwd_lcores_config();
 590         set_def_peer_eth_addrs();
 591         set_default_fwd_ports_config();
 592 }
 593
 594 /* extremely pessimistic estimation of memory required to create a mempool */
 595 static int
 596 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 597 {
 598         unsigned int n_pages, mbuf_per_pg, leftover;
 599         uint64_t total_mem, mbuf_mem, obj_sz;
 600
 601         /* there is no good way to predict how much space the mempool will
 602          * occupy because it will allocate chunks on the fly, and some of those
 603          * will come from default DPDK memory while some will come from our
 604          * external memory, so just assume 128MB will be enough for everyone.
 605          */
 606         uint64_t hdr_mem = 128 << 20;
 607
 608         /* account for possible non-contiguousness */
 609         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 610         if (obj_sz > pgsz) {
 611                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 612                 return -1;
 613         }
 614
 615         mbuf_per_pg = pgsz / obj_sz;
 616         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 617         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 618
 619         mbuf_mem = n_pages * pgsz;
 620
 621         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 622
 623         if (total_mem > SIZE_MAX) {
 624                 TESTPMD_LOG(ERR, "Memory size too big\n");
 625                 return -1;
 626         }
 627         *out = (size_t)total_mem;
 628
 629         return 0;
 630 }
 631
 632 static int
 633 pagesz_flags(uint64_t page_sz)
 634 {
 635         /* as per mmap() manpage, all page sizes are log2 of page size
 636          * shifted by MAP_HUGE_SHIFT
 637          */
 638         int log2 = rte_log2_u64(page_sz);
 639
 640         return (log2 << HUGE_SHIFT);
 641 }
 642
 643 static void *
 644 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 645 {
 646         void *addr;
 647         int flags;
 648
 649         /* allocate anonymous hugepages */
 650         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 651         if (huge)
 652                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 653
 654         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 655         if (addr == MAP_FAILED)
 656                 return NULL;
 657
 658         return addr;
 659 }
 660
 661 struct extmem_param {
 662         void *addr;
 663         size_t len;
 664         size_t pgsz;
 665         rte_iova_t *iova_table;
 666         unsigned int iova_table_len;
 667 };
 668
 669 static int
 670 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 671                 bool huge)
 672 {
 673         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 674                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 675         unsigned int cur_page, n_pages, pgsz_idx;
 676         size_t mem_sz, cur_pgsz;
 677         rte_iova_t *iovas = NULL;
 678         void *addr;
 679         int ret;
 680
 681         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 682                 /* skip anything that is too big */
 683                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 684                         continue;
 685
 686                 cur_pgsz = pgsizes[pgsz_idx];
 687
 688                 /* if we were told not to allocate hugepages, override */
 689                 if (!huge)
 690                         cur_pgsz = sysconf(_SC_PAGESIZE);
 691
 692                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 693                 if (ret < 0) {
 694                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 695                         return -1;
 696                 }
 697
 698                 /* allocate our memory */
 699                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 700
 701                 /* if we couldn't allocate memory with a specified page size,
 702                  * that doesn't mean we can't do it with other page sizes, so
 703                  * try another one.
 704                  */
 705                 if (addr == NULL)
 706                         continue;
 707
 708                 /* store IOVA addresses for every page in this memory area */
 709                 n_pages = mem_sz / cur_pgsz;
 710
 711                 iovas = malloc(sizeof(*iovas) * n_pages);
 712
 713                 if (iovas == NULL) {
 714                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 715                         goto fail;
 716                 }
 717                 /* lock memory if it's not huge pages */
 718                 if (!huge)
 719                         mlock(addr, mem_sz);
 720
 721                 /* populate IOVA addresses */
 722                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 723                         rte_iova_t iova;
 724                         size_t offset;
 725                         void *cur;
 726
 727                         offset = cur_pgsz * cur_page;
 728                         cur = RTE_PTR_ADD(addr, offset);
 729
 730                         /* touch the page before getting its IOVA */
 731                         *(volatile char *)cur = 0;
 732
 733                         iova = rte_mem_virt2iova(cur);
 734
 735                         iovas[cur_page] = iova;
 736                 }
 737
 738                 break;
 739         }
 740         /* if we couldn't allocate anything */
 741         if (iovas == NULL)
 742                 return -1;
 743
 744         param->addr = addr;
 745         param->len = mem_sz;
 746         param->pgsz = cur_pgsz;
 747         param->iova_table = iovas;
 748         param->iova_table_len = n_pages;
 749
 750         return 0;
 751 fail:
 752         if (iovas)
 753                 free(iovas);
 754         if (addr)
 755                 munmap(addr, mem_sz);
 756
 757         return -1;
 758 }
 759
 760 static int
 761 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 762 {
 763         struct extmem_param param;
 764         int socket_id, ret;
 765
 766         memset(&param, 0, sizeof(param));
 767
 768         /* check if our heap exists */
 769         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 770         if (socket_id < 0) {
 771                 /* create our heap */
 772                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 773                 if (ret < 0) {
 774                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 775                         return -1;
 776                 }
 777         }
 778
 779         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 780         if (ret < 0) {
 781                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 782                 return -1;
 783         }
 784
 785         /* we now have a valid memory area, so add it to heap */
 786         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 787                         param.addr, param.len, param.iova_table,
 788                         param.iova_table_len, param.pgsz);
 789
 790         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 791
 792         /* not needed any more */
 793         free(param.iova_table);
 794
 795         if (ret < 0) {
 796                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 797                 munmap(param.addr, param.len);
 798                 return -1;
 799         }
 800
 801         /* success */
 802
 803         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 804                         param.len >> 20);
 805
 806         return 0;
 807 }
 808 static void
 809 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 810              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 811 {
 812         uint16_t pid = 0;
 813         int ret;
 814
 815         RTE_ETH_FOREACH_DEV(pid) {
 816                 struct rte_eth_dev *dev =
 817                         &rte_eth_devices[pid];
 818
 819                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 820                                         memhdr->len);
 821                 if (ret) {
 822                         TESTPMD_LOG(DEBUG,
 823                                     "unable to DMA unmap addr 0x%p "
 824                                     "for device %s\n",
 825                                     memhdr->addr, dev->data->name);
 826                 }
 827         }
 828         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 829         if (ret) {
 830                 TESTPMD_LOG(DEBUG,
 831                             "unable to un-register addr 0x%p\n", memhdr->addr);
 832         }
 833 }
 834
 835 static void
 836 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 837            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 838 {
 839         uint16_t pid = 0;
 840         size_t page_size = sysconf(_SC_PAGESIZE);
 841         int ret;
 842
 843         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 844                                   page_size);
 845         if (ret) {
 846                 TESTPMD_LOG(DEBUG,
 847                             "unable to register addr 0x%p\n", memhdr->addr);
 848                 return;
 849         }
 850         RTE_ETH_FOREACH_DEV(pid) {
 851                 struct rte_eth_dev *dev =
 852                         &rte_eth_devices[pid];
 853
 854                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 855                                       memhdr->len);
 856                 if (ret) {
 857                         TESTPMD_LOG(DEBUG,
 858                                     "unable to DMA map addr 0x%p "
 859                                     "for device %s\n",
 860                                     memhdr->addr, dev->data->name);
 861                 }
 862         }
 863 }
 864
 865 /*
 866  * Configuration initialisation done once at init time.
 867  */
 868 static struct rte_mempool *
 869 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 870                  unsigned int socket_id)
 871 {
 872         char pool_name[RTE_MEMPOOL_NAMESIZE];
 873         struct rte_mempool *rte_mp = NULL;
 874         uint32_t mb_size;
 875
 876         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 877         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 878
 879         TESTPMD_LOG(INFO,
 880                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 881                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 882
 883         switch (mp_alloc_type) {
 884         case MP_ALLOC_NATIVE:
 885                 {
 886                         /* wrapper to rte_mempool_create() */
 887                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 888                                         rte_mbuf_best_mempool_ops());
 889                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 890                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 891                         break;
 892                 }
 893         case MP_ALLOC_ANON:
 894                 {
 895                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 896                                 mb_size, (unsigned int) mb_mempool_cache,
 897                                 sizeof(struct rte_pktmbuf_pool_private),
 898                                 socket_id, mempool_flags);
 899                         if (rte_mp == NULL)
 900                                 goto err;
 901
 902                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 903                                 rte_mempool_free(rte_mp);
 904                                 rte_mp = NULL;
 905                                 goto err;
 906                         }
 907                         rte_pktmbuf_pool_init(rte_mp, NULL);
 908                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 909                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 910                         break;
 911                 }
 912         case MP_ALLOC_XMEM:
 913         case MP_ALLOC_XMEM_HUGE:
 914                 {
 915                         int heap_socket;
 916                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 917
 918                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 919                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 920
 921                         heap_socket =
 922                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 923                         if (heap_socket < 0)
 924                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 925
 926                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 927                                         rte_mbuf_best_mempool_ops());
 928                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 929                                         mb_mempool_cache, 0, mbuf_seg_size,
 930                                         heap_socket);
 931                         break;
 932                 }
 933         default:
 934                 {
 935                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 936                 }
 937         }
 938
 939 err:
 940         if (rte_mp == NULL) {
 941                 rte_exit(EXIT_FAILURE,
 942                         "Creation of mbuf pool for socket %u failed: %s\n",
 943                         socket_id, rte_strerror(rte_errno));
 944         } else if (verbose_level > 0) {
 945                 rte_mempool_dump(stdout, rte_mp);
 946         }
 947         return rte_mp;
 948 }
 949
 950 /*
 951  * Check given socket id is valid or not with NUMA mode,
 952  * if valid, return 0, else return -1
 953  */
 954 static int
 955 check_socket_id(const unsigned int socket_id)
 956 {
 957         static int warning_once = 0;
 958
 959         if (new_socket_id(socket_id)) {
 960                 if (!warning_once && numa_support)
 961                         printf("Warning: NUMA should be configured manually by"
 962                                " using --port-numa-config and"
 963                                " --ring-numa-config parameters along with"
 964                                " --numa.\n");
 965                 warning_once = 1;
 966                 return -1;
 967         }
 968         return 0;
 969 }
 970
 971 /*
 972  * Get the allowed maximum number of RX queues.
 973  * *pid return the port id which has minimal value of
 974  * max_rx_queues in all ports.
 975  */
 976 queueid_t
 977 get_allowed_max_nb_rxq(portid_t *pid)
 978 {
 979         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
 980         bool max_rxq_valid = false;
 981         portid_t pi;
 982         struct rte_eth_dev_info dev_info;
 983
 984         RTE_ETH_FOREACH_DEV(pi) {
 985                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
 986                         continue;
 987
 988                 max_rxq_valid = true;
 989                 if (dev_info.max_rx_queues < allowed_max_rxq) {
 990                         allowed_max_rxq = dev_info.max_rx_queues;
 991                         *pid = pi;
 992                 }
 993         }
 994         return max_rxq_valid ? allowed_max_rxq : 0;
 995 }
 996
 997 /*
 998  * Check input rxq is valid or not.
 999  * If input rxq is not greater than any of maximum number
1000  * of RX queues of all ports, it is valid.
1001  * if valid, return 0, else return -1
1002  */
1003 int
1004 check_nb_rxq(queueid_t rxq)
1005 {
1006         queueid_t allowed_max_rxq;
1007         portid_t pid = 0;
1008
1009         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1010         if (rxq > allowed_max_rxq) {
1011                 printf("Fail: input rxq (%u) can't be greater "
1012                        "than max_rx_queues (%u) of port %u\n",
1013                        rxq,
1014                        allowed_max_rxq,
1015                        pid);
1016                 return -1;
1017         }
1018         return 0;
1019 }
1020
1021 /*
1022  * Get the allowed maximum number of TX queues.
1023  * *pid return the port id which has minimal value of
1024  * max_tx_queues in all ports.
1025  */
1026 queueid_t
1027 get_allowed_max_nb_txq(portid_t *pid)
1028 {
1029         queueid_t allowed_max_txq = MAX_QUEUE_ID;
1030         bool max_txq_valid = false;
1031         portid_t pi;
1032         struct rte_eth_dev_info dev_info;
1033
1034         RTE_ETH_FOREACH_DEV(pi) {
1035                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1036                         continue;
1037
1038                 max_txq_valid = true;
1039                 if (dev_info.max_tx_queues < allowed_max_txq) {
1040                         allowed_max_txq = dev_info.max_tx_queues;
1041                         *pid = pi;
1042                 }
1043         }
1044         return max_txq_valid ? allowed_max_txq : 0;
1045 }
1046
1047 /*
1048  * Check input txq is valid or not.
1049  * If input txq is not greater than any of maximum number
1050  * of TX queues of all ports, it is valid.
1051  * if valid, return 0, else return -1
1052  */
1053 int
1054 check_nb_txq(queueid_t txq)
1055 {
1056         queueid_t allowed_max_txq;
1057         portid_t pid = 0;
1058
1059         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1060         if (txq > allowed_max_txq) {
1061                 printf("Fail: input txq (%u) can't be greater "
1062                        "than max_tx_queues (%u) of port %u\n",
1063                        txq,
1064                        allowed_max_txq,
1065                        pid);
1066                 return -1;
1067         }
1068         return 0;
1069 }
1070
1071 /*
1072  * Get the allowed maximum number of hairpin queues.
1073  * *pid return the port id which has minimal value of
1074  * max_hairpin_queues in all ports.
1075  */
1076 queueid_t
1077 get_allowed_max_nb_hairpinq(portid_t *pid)
1078 {
1079         queueid_t allowed_max_hairpinq = MAX_QUEUE_ID;
1080         portid_t pi;
1081         struct rte_eth_hairpin_cap cap;
1082
1083         RTE_ETH_FOREACH_DEV(pi) {
1084                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1085                         *pid = pi;
1086                         return 0;
1087                 }
1088                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1089                         allowed_max_hairpinq = cap.max_nb_queues;
1090                         *pid = pi;
1091                 }
1092         }
1093         return allowed_max_hairpinq;
1094 }
1095
1096 /*
1097  * Check input hairpin is valid or not.
1098  * If input hairpin is not greater than any of maximum number
1099  * of hairpin queues of all ports, it is valid.
1100  * if valid, return 0, else return -1
1101  */
1102 int
1103 check_nb_hairpinq(queueid_t hairpinq)
1104 {
1105         queueid_t allowed_max_hairpinq;
1106         portid_t pid = 0;
1107
1108         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1109         if (hairpinq > allowed_max_hairpinq) {
1110                 printf("Fail: input hairpin (%u) can't be greater "
1111                        "than max_hairpin_queues (%u) of port %u\n",
1112                        hairpinq, allowed_max_hairpinq, pid);
1113                 return -1;
1114         }
1115         return 0;
1116 }
1117
1118 static void
1119 init_config(void)
1120 {
1121         portid_t pid;
1122         struct rte_port *port;
1123         struct rte_mempool *mbp;
1124         unsigned int nb_mbuf_per_pool;
1125         lcoreid_t  lc_id;
1126         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1127         struct rte_gro_param gro_param;
1128         uint32_t gso_types;
1129         uint16_t data_size;
1130         bool warning = 0;
1131         int k;
1132         int ret;
1133
1134         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1135
1136         /* Configuration of logical cores. */
1137         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1138                                 sizeof(struct fwd_lcore *) * nb_lcores,
1139                                 RTE_CACHE_LINE_SIZE);
1140         if (fwd_lcores == NULL) {
1141                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1142                                                         "failed\n", nb_lcores);
1143         }
1144         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1145                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1146                                                sizeof(struct fwd_lcore),
1147                                                RTE_CACHE_LINE_SIZE);
1148                 if (fwd_lcores[lc_id] == NULL) {
1149                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1150                                                                 "failed\n");
1151                 }
1152                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1153         }
1154
1155         RTE_ETH_FOREACH_DEV(pid) {
1156                 port = &ports[pid];
1157                 /* Apply default TxRx configuration for all ports */
1158                 port->dev_conf.txmode = tx_mode;
1159                 port->dev_conf.rxmode = rx_mode;
1160
1161                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1162                 if (ret != 0)
1163                         rte_exit(EXIT_FAILURE,
1164                                  "rte_eth_dev_info_get() failed\n");
1165
1166                 if (!(port->dev_info.tx_offload_capa &
1167                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1168                         port->dev_conf.txmode.offloads &=
1169                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1170                 if (numa_support) {
1171                         if (port_numa[pid] != NUMA_NO_CONFIG)
1172                                 port_per_socket[port_numa[pid]]++;
1173                         else {
1174                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1175
1176                                 /*
1177                                  * if socket_id is invalid,
1178                                  * set to the first available socket.
1179                                  */
1180                                 if (check_socket_id(socket_id) < 0)
1181                                         socket_id = socket_ids[0];
1182                                 port_per_socket[socket_id]++;
1183                         }
1184                 }
1185
1186                 /* Apply Rx offloads configuration */
1187                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1188                         port->rx_conf[k].offloads =
1189                                 port->dev_conf.rxmode.offloads;
1190                 /* Apply Tx offloads configuration */
1191                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1192                         port->tx_conf[k].offloads =
1193                                 port->dev_conf.txmode.offloads;
1194
1195                 /* set flag to initialize port/queue */
1196                 port->need_reconfig = 1;
1197                 port->need_reconfig_queues = 1;
1198                 port->tx_metadata = 0;
1199
1200                 /* Check for maximum number of segments per MTU. Accordingly
1201                  * update the mbuf data size.
1202                  */
1203                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1204                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1205                         data_size = rx_mode.max_rx_pkt_len /
1206                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1207
1208                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1209                                                         mbuf_data_size) {
1210                                 mbuf_data_size = data_size +
1211                                                  RTE_PKTMBUF_HEADROOM;
1212                                 warning = 1;
1213                         }
1214                 }
1215         }
1216
1217         if (warning)
1218                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1219                             mbuf_data_size);
1220
1221         /*
1222          * Create pools of mbuf.
1223          * If NUMA support is disabled, create a single pool of mbuf in
1224          * socket 0 memory by default.
1225          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1226          *
1227          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1228          * nb_txd can be configured at run time.
1229          */
1230         if (param_total_num_mbufs)
1231                 nb_mbuf_per_pool = param_total_num_mbufs;
1232         else {
1233                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1234                         (nb_lcores * mb_mempool_cache) +
1235                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1236                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1237         }
1238
1239         if (numa_support) {
1240                 uint8_t i;
1241
1242                 for (i = 0; i < num_sockets; i++)
1243                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1244                                                        nb_mbuf_per_pool,
1245                                                        socket_ids[i]);
1246         } else {
1247                 if (socket_num == UMA_NO_CONFIG)
1248                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1249                                                        nb_mbuf_per_pool, 0);
1250                 else
1251                         mempools[socket_num] = mbuf_pool_create
1252                                                         (mbuf_data_size,
1253                                                          nb_mbuf_per_pool,
1254                                                          socket_num);
1255         }
1256
1257         init_port_config();
1258
1259         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1260                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1261         /*
1262          * Records which Mbuf pool to use by each logical core, if needed.
1263          */
1264         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1265                 mbp = mbuf_pool_find(
1266                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1267
1268                 if (mbp == NULL)
1269                         mbp = mbuf_pool_find(0);
1270                 fwd_lcores[lc_id]->mbp = mbp;
1271                 /* initialize GSO context */
1272                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1273                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1274                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1275                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1276                         RTE_ETHER_CRC_LEN;
1277                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1278         }
1279
1280         /* Configuration of packet forwarding streams. */
1281         if (init_fwd_streams() < 0)
1282                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1283
1284         fwd_config_setup();
1285
1286         /* create a gro context for each lcore */
1287         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1288         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1289         gro_param.max_item_per_flow = MAX_PKT_BURST;
1290         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1291                 gro_param.socket_id = rte_lcore_to_socket_id(
1292                                 fwd_lcores_cpuids[lc_id]);
1293                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1294                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1295                         rte_exit(EXIT_FAILURE,
1296                                         "rte_gro_ctx_create() failed\n");
1297                 }
1298         }
1299
1300 #if defined RTE_LIBRTE_PMD_SOFTNIC
1301         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1302                 RTE_ETH_FOREACH_DEV(pid) {
1303                         port = &ports[pid];
1304                         const char *driver = port->dev_info.driver_name;
1305
1306                         if (strcmp(driver, "net_softnic") == 0)
1307                                 port->softport.fwd_lcore_arg = fwd_lcores;
1308                 }
1309         }
1310 #endif
1311
1312 }
1313
1314
1315 void
1316 reconfig(portid_t new_port_id, unsigned socket_id)
1317 {
1318         struct rte_port *port;
1319         int ret;
1320
1321         /* Reconfiguration of Ethernet ports. */
1322         port = &ports[new_port_id];
1323
1324         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1325         if (ret != 0)
1326                 return;
1327
1328         /* set flag to initialize port/queue */
1329         port->need_reconfig = 1;
1330         port->need_reconfig_queues = 1;
1331         port->socket_id = socket_id;
1332
1333         init_port_config();
1334 }
1335
1336
1337 int
1338 init_fwd_streams(void)
1339 {
1340         portid_t pid;
1341         struct rte_port *port;
1342         streamid_t sm_id, nb_fwd_streams_new;
1343         queueid_t q;
1344
1345         /* set socket id according to numa or not */
1346         RTE_ETH_FOREACH_DEV(pid) {
1347                 port = &ports[pid];
1348                 if (nb_rxq > port->dev_info.max_rx_queues) {
1349                         printf("Fail: nb_rxq(%d) is greater than "
1350                                 "max_rx_queues(%d)\n", nb_rxq,
1351                                 port->dev_info.max_rx_queues);
1352                         return -1;
1353                 }
1354                 if (nb_txq > port->dev_info.max_tx_queues) {
1355                         printf("Fail: nb_txq(%d) is greater than "
1356                                 "max_tx_queues(%d)\n", nb_txq,
1357                                 port->dev_info.max_tx_queues);
1358                         return -1;
1359                 }
1360                 if (numa_support) {
1361                         if (port_numa[pid] != NUMA_NO_CONFIG)
1362                                 port->socket_id = port_numa[pid];
1363                         else {
1364                                 port->socket_id = rte_eth_dev_socket_id(pid);
1365
1366                                 /*
1367                                  * if socket_id is invalid,
1368                                  * set to the first available socket.
1369                                  */
1370                                 if (check_socket_id(port->socket_id) < 0)
1371                                         port->socket_id = socket_ids[0];
1372                         }
1373                 }
1374                 else {
1375                         if (socket_num == UMA_NO_CONFIG)
1376                                 port->socket_id = 0;
1377                         else
1378                                 port->socket_id = socket_num;
1379                 }
1380         }
1381
1382         q = RTE_MAX(nb_rxq, nb_txq);
1383         if (q == 0) {
1384                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1385                 return -1;
1386         }
1387         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1388         if (nb_fwd_streams_new == nb_fwd_streams)
1389                 return 0;
1390         /* clear the old */
1391         if (fwd_streams != NULL) {
1392                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1393                         if (fwd_streams[sm_id] == NULL)
1394                                 continue;
1395                         rte_free(fwd_streams[sm_id]);
1396                         fwd_streams[sm_id] = NULL;
1397                 }
1398                 rte_free(fwd_streams);
1399                 fwd_streams = NULL;
1400         }
1401
1402         /* init new */
1403         nb_fwd_streams = nb_fwd_streams_new;
1404         if (nb_fwd_streams) {
1405                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1406                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1407                         RTE_CACHE_LINE_SIZE);
1408                 if (fwd_streams == NULL)
1409                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1410                                  " (struct fwd_stream *)) failed\n",
1411                                  nb_fwd_streams);
1412
1413                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1414                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1415                                 " struct fwd_stream", sizeof(struct fwd_stream),
1416                                 RTE_CACHE_LINE_SIZE);
1417                         if (fwd_streams[sm_id] == NULL)
1418                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1419                                          "(struct fwd_stream) failed\n");
1420                 }
1421         }
1422
1423         return 0;
1424 }
1425
1426 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1427 static void
1428 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1429 {
1430         unsigned int total_burst;
1431         unsigned int nb_burst;
1432         unsigned int burst_stats[3];
1433         uint16_t pktnb_stats[3];
1434         uint16_t nb_pkt;
1435         int burst_percent[3];
1436
1437         /*
1438          * First compute the total number of packet bursts and the
1439          * two highest numbers of bursts of the same number of packets.
1440          */
1441         total_burst = 0;
1442         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1443         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1444         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1445                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1446                 if (nb_burst == 0)
1447                         continue;
1448                 total_burst += nb_burst;
1449                 if (nb_burst > burst_stats[0]) {
1450                         burst_stats[1] = burst_stats[0];
1451                         pktnb_stats[1] = pktnb_stats[0];
1452                         burst_stats[0] = nb_burst;
1453                         pktnb_stats[0] = nb_pkt;
1454                 } else if (nb_burst > burst_stats[1]) {
1455                         burst_stats[1] = nb_burst;
1456                         pktnb_stats[1] = nb_pkt;
1457                 }
1458         }
1459         if (total_burst == 0)
1460                 return;
1461         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1462         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1463                burst_percent[0], (int) pktnb_stats[0]);
1464         if (burst_stats[0] == total_burst) {
1465                 printf("]\n");
1466                 return;
1467         }
1468         if (burst_stats[0] + burst_stats[1] == total_burst) {
1469                 printf(" + %d%% of %d pkts]\n",
1470                        100 - burst_percent[0], pktnb_stats[1]);
1471                 return;
1472         }
1473         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1474         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1475         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1476                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1477                 return;
1478         }
1479         printf(" + %d%% of %d pkts + %d%% of others]\n",
1480                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1481 }
1482 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1483
1484 static void
1485 fwd_stream_stats_display(streamid_t stream_id)
1486 {
1487         struct fwd_stream *fs;
1488         static const char *fwd_top_stats_border = "-------";
1489
1490         fs = fwd_streams[stream_id];
1491         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1492             (fs->fwd_dropped == 0))
1493                 return;
1494         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1495                "TX Port=%2d/Queue=%2d %s\n",
1496                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1497                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1498         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1499                " TX-dropped: %-14"PRIu64,
1500                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1501
1502         /* if checksum mode */
1503         if (cur_fwd_eng == &csum_fwd_engine) {
1504                 printf("  RX- bad IP checksum: %-14"PRIu64
1505                        "  Rx- bad L4 checksum: %-14"PRIu64
1506                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1507                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1508                         fs->rx_bad_outer_l4_csum);
1509         } else {
1510                 printf("\n");
1511         }
1512
1513 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1514         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1515         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1516 #endif
1517 }
1518
1519 void
1520 fwd_stats_display(void)
1521 {
1522         static const char *fwd_stats_border = "----------------------";
1523         static const char *acc_stats_border = "+++++++++++++++";
1524         struct {
1525                 struct fwd_stream *rx_stream;
1526                 struct fwd_stream *tx_stream;
1527                 uint64_t tx_dropped;
1528                 uint64_t rx_bad_ip_csum;
1529                 uint64_t rx_bad_l4_csum;
1530                 uint64_t rx_bad_outer_l4_csum;
1531         } ports_stats[RTE_MAX_ETHPORTS];
1532         uint64_t total_rx_dropped = 0;
1533         uint64_t total_tx_dropped = 0;
1534         uint64_t total_rx_nombuf = 0;
1535         struct rte_eth_stats stats;
1536 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1537         uint64_t fwd_cycles = 0;
1538 #endif
1539         uint64_t total_recv = 0;
1540         uint64_t total_xmit = 0;
1541         struct rte_port *port;
1542         streamid_t sm_id;
1543         portid_t pt_id;
1544         int i;
1545
1546         memset(ports_stats, 0, sizeof(ports_stats));
1547
1548         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1549                 struct fwd_stream *fs = fwd_streams[sm_id];
1550
1551                 if (cur_fwd_config.nb_fwd_streams >
1552                     cur_fwd_config.nb_fwd_ports) {
1553                         fwd_stream_stats_display(sm_id);
1554                 } else {
1555                         ports_stats[fs->tx_port].tx_stream = fs;
1556                         ports_stats[fs->rx_port].rx_stream = fs;
1557                 }
1558
1559                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1560
1561                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1562                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1563                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1564                                 fs->rx_bad_outer_l4_csum;
1565
1566 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1567                 fwd_cycles += fs->core_cycles;
1568 #endif
1569         }
1570         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1571                 uint8_t j;
1572
1573                 pt_id = fwd_ports_ids[i];
1574                 port = &ports[pt_id];
1575
1576                 rte_eth_stats_get(pt_id, &stats);
1577                 stats.ipackets -= port->stats.ipackets;
1578                 stats.opackets -= port->stats.opackets;
1579                 stats.ibytes -= port->stats.ibytes;
1580                 stats.obytes -= port->stats.obytes;
1581                 stats.imissed -= port->stats.imissed;
1582                 stats.oerrors -= port->stats.oerrors;
1583                 stats.rx_nombuf -= port->stats.rx_nombuf;
1584
1585                 total_recv += stats.ipackets;
1586                 total_xmit += stats.opackets;
1587                 total_rx_dropped += stats.imissed;
1588                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1589                 total_tx_dropped += stats.oerrors;
1590                 total_rx_nombuf  += stats.rx_nombuf;
1591
1592                 printf("\n  %s Forward statistics for port %-2d %s\n",
1593                        fwd_stats_border, pt_id, fwd_stats_border);
1594
1595                 if (!port->rx_queue_stats_mapping_enabled &&
1596                     !port->tx_queue_stats_mapping_enabled) {
1597                         printf("  RX-packets: %-14"PRIu64
1598                                " RX-dropped: %-14"PRIu64
1599                                "RX-total: %-"PRIu64"\n",
1600                                stats.ipackets, stats.imissed,
1601                                stats.ipackets + stats.imissed);
1602
1603                         if (cur_fwd_eng == &csum_fwd_engine)
1604                                 printf("  Bad-ipcsum: %-14"PRIu64
1605                                        " Bad-l4csum: %-14"PRIu64
1606                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1607                                        ports_stats[pt_id].rx_bad_ip_csum,
1608                                        ports_stats[pt_id].rx_bad_l4_csum,
1609                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1610                         if (stats.ierrors + stats.rx_nombuf > 0) {
1611                                 printf("  RX-error: %-"PRIu64"\n",
1612                                        stats.ierrors);
1613                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1614                                        stats.rx_nombuf);
1615                         }
1616
1617                         printf("  TX-packets: %-14"PRIu64
1618                                " TX-dropped: %-14"PRIu64
1619                                "TX-total: %-"PRIu64"\n",
1620                                stats.opackets, ports_stats[pt_id].tx_dropped,
1621                                stats.opackets + ports_stats[pt_id].tx_dropped);
1622                 } else {
1623                         printf("  RX-packets:             %14"PRIu64
1624                                "    RX-dropped:%14"PRIu64
1625                                "    RX-total:%14"PRIu64"\n",
1626                                stats.ipackets, stats.imissed,
1627                                stats.ipackets + stats.imissed);
1628
1629                         if (cur_fwd_eng == &csum_fwd_engine)
1630                                 printf("  Bad-ipcsum:%14"PRIu64
1631                                        "    Bad-l4csum:%14"PRIu64
1632                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1633                                        ports_stats[pt_id].rx_bad_ip_csum,
1634                                        ports_stats[pt_id].rx_bad_l4_csum,
1635                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1636                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1637                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1638                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1639                                        stats.rx_nombuf);
1640                         }
1641
1642                         printf("  TX-packets:             %14"PRIu64
1643                                "    TX-dropped:%14"PRIu64
1644                                "    TX-total:%14"PRIu64"\n",
1645                                stats.opackets, ports_stats[pt_id].tx_dropped,
1646                                stats.opackets + ports_stats[pt_id].tx_dropped);
1647                 }
1648
1649 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1650                 if (ports_stats[pt_id].rx_stream)
1651                         pkt_burst_stats_display("RX",
1652                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1653                 if (ports_stats[pt_id].tx_stream)
1654                         pkt_burst_stats_display("TX",
1655                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1656 #endif
1657
1658                 if (port->rx_queue_stats_mapping_enabled) {
1659                         printf("\n");
1660                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1661                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1662                                        "     RX-errors:%14"PRIu64
1663                                        "    RX-bytes:%14"PRIu64"\n",
1664                                        j, stats.q_ipackets[j],
1665                                        stats.q_errors[j], stats.q_ibytes[j]);
1666                         }
1667                         printf("\n");
1668                 }
1669                 if (port->tx_queue_stats_mapping_enabled) {
1670                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1671                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1672                                        "                                 TX-bytes:%14"
1673                                        PRIu64"\n",
1674                                        j, stats.q_opackets[j],
1675                                        stats.q_obytes[j]);
1676                         }
1677                 }
1678
1679                 printf("  %s--------------------------------%s\n",
1680                        fwd_stats_border, fwd_stats_border);
1681         }
1682
1683         printf("\n  %s Accumulated forward statistics for all ports"
1684                "%s\n",
1685                acc_stats_border, acc_stats_border);
1686         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1687                "%-"PRIu64"\n"
1688                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1689                "%-"PRIu64"\n",
1690                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1691                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1692         if (total_rx_nombuf > 0)
1693                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1694         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1695                "%s\n",
1696                acc_stats_border, acc_stats_border);
1697 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1698         if (total_recv > 0)
1699                 printf("\n  CPU cycles/packet=%u (total cycles="
1700                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1701                        (unsigned int)(fwd_cycles / total_recv),
1702                        fwd_cycles, total_recv);
1703 #endif
1704 }
1705
1706 void
1707 fwd_stats_reset(void)
1708 {
1709         streamid_t sm_id;
1710         portid_t pt_id;
1711         int i;
1712
1713         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1714                 pt_id = fwd_ports_ids[i];
1715                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1716         }
1717         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1718                 struct fwd_stream *fs = fwd_streams[sm_id];
1719
1720                 fs->rx_packets = 0;
1721                 fs->tx_packets = 0;
1722                 fs->fwd_dropped = 0;
1723                 fs->rx_bad_ip_csum = 0;
1724                 fs->rx_bad_l4_csum = 0;
1725                 fs->rx_bad_outer_l4_csum = 0;
1726
1727 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1728                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1729                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1730 #endif
1731 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1732                 fs->core_cycles = 0;
1733 #endif
1734         }
1735 }
1736
1737 static void
1738 flush_fwd_rx_queues(void)
1739 {
1740         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1741         portid_t  rxp;
1742         portid_t port_id;
1743         queueid_t rxq;
1744         uint16_t  nb_rx;
1745         uint16_t  i;
1746         uint8_t   j;
1747         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1748         uint64_t timer_period;
1749
1750         /* convert to number of cycles */
1751         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1752
1753         for (j = 0; j < 2; j++) {
1754                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1755                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1756                                 port_id = fwd_ports_ids[rxp];
1757                                 /**
1758                                 * testpmd can stuck in the below do while loop
1759                                 * if rte_eth_rx_burst() always returns nonzero
1760                                 * packets. So timer is added to exit this loop
1761                                 * after 1sec timer expiry.
1762                                 */
1763                                 prev_tsc = rte_rdtsc();
1764                                 do {
1765                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1766                                                 pkts_burst, MAX_PKT_BURST);
1767                                         for (i = 0; i < nb_rx; i++)
1768                                                 rte_pktmbuf_free(pkts_burst[i]);
1769
1770                                         cur_tsc = rte_rdtsc();
1771                                         diff_tsc = cur_tsc - prev_tsc;
1772                                         timer_tsc += diff_tsc;
1773                                 } while ((nb_rx > 0) &&
1774                                         (timer_tsc < timer_period));
1775                                 timer_tsc = 0;
1776                         }
1777                 }
1778                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1779         }
1780 }
1781
1782 static void
1783 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1784 {
1785         struct fwd_stream **fsm;
1786         streamid_t nb_fs;
1787         streamid_t sm_id;
1788 #ifdef RTE_LIBRTE_BITRATE
1789         uint64_t tics_per_1sec;
1790         uint64_t tics_datum;
1791         uint64_t tics_current;
1792         uint16_t i, cnt_ports;
1793
1794         cnt_ports = nb_ports;
1795         tics_datum = rte_rdtsc();
1796         tics_per_1sec = rte_get_timer_hz();
1797 #endif
1798         fsm = &fwd_streams[fc->stream_idx];
1799         nb_fs = fc->stream_nb;
1800         do {
1801                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1802                         (*pkt_fwd)(fsm[sm_id]);
1803 #ifdef RTE_LIBRTE_BITRATE
1804                 if (bitrate_enabled != 0 &&
1805                                 bitrate_lcore_id == rte_lcore_id()) {
1806                         tics_current = rte_rdtsc();
1807                         if (tics_current - tics_datum >= tics_per_1sec) {
1808                                 /* Periodic bitrate calculation */
1809                                 for (i = 0; i < cnt_ports; i++)
1810                                         rte_stats_bitrate_calc(bitrate_data,
1811                                                 ports_ids[i]);
1812                                 tics_datum = tics_current;
1813                         }
1814                 }
1815 #endif
1816 #ifdef RTE_LIBRTE_LATENCY_STATS
1817                 if (latencystats_enabled != 0 &&
1818                                 latencystats_lcore_id == rte_lcore_id())
1819                         rte_latencystats_update();
1820 #endif
1821
1822         } while (! fc->stopped);
1823 }
1824
1825 static int
1826 start_pkt_forward_on_core(void *fwd_arg)
1827 {
1828         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1829                              cur_fwd_config.fwd_eng->packet_fwd);
1830         return 0;
1831 }
1832
1833 /*
1834  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1835  * Used to start communication flows in network loopback test configurations.
1836  */
1837 static int
1838 run_one_txonly_burst_on_core(void *fwd_arg)
1839 {
1840         struct fwd_lcore *fwd_lc;
1841         struct fwd_lcore tmp_lcore;
1842
1843         fwd_lc = (struct fwd_lcore *) fwd_arg;
1844         tmp_lcore = *fwd_lc;
1845         tmp_lcore.stopped = 1;
1846         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1847         return 0;
1848 }
1849
1850 /*
1851  * Launch packet forwarding:
1852  *     - Setup per-port forwarding context.
1853  *     - launch logical cores with their forwarding configuration.
1854  */
1855 static void
1856 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1857 {
1858         port_fwd_begin_t port_fwd_begin;
1859         unsigned int i;
1860         unsigned int lc_id;
1861         int diag;
1862
1863         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1864         if (port_fwd_begin != NULL) {
1865                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1866                         (*port_fwd_begin)(fwd_ports_ids[i]);
1867         }
1868         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1869                 lc_id = fwd_lcores_cpuids[i];
1870                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1871                         fwd_lcores[i]->stopped = 0;
1872                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1873                                                      fwd_lcores[i], lc_id);
1874                         if (diag != 0)
1875                                 printf("launch lcore %u failed - diag=%d\n",
1876                                        lc_id, diag);
1877                 }
1878         }
1879 }
1880
1881 /*
1882  * Launch packet forwarding configuration.
1883  */
1884 void
1885 start_packet_forwarding(int with_tx_first)
1886 {
1887         port_fwd_begin_t port_fwd_begin;
1888         port_fwd_end_t  port_fwd_end;
1889         struct rte_port *port;
1890         unsigned int i;
1891         portid_t   pt_id;
1892
1893         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1894                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1895
1896         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1897                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1898
1899         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1900                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1901                 (!nb_rxq || !nb_txq))
1902                 rte_exit(EXIT_FAILURE,
1903                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1904                         cur_fwd_eng->fwd_mode_name);
1905
1906         if (all_ports_started() == 0) {
1907                 printf("Not all ports were started\n");
1908                 return;
1909         }
1910         if (test_done == 0) {
1911                 printf("Packet forwarding already started\n");
1912                 return;
1913         }
1914
1915
1916         if(dcb_test) {
1917                 for (i = 0; i < nb_fwd_ports; i++) {
1918                         pt_id = fwd_ports_ids[i];
1919                         port = &ports[pt_id];
1920                         if (!port->dcb_flag) {
1921                                 printf("In DCB mode, all forwarding ports must "
1922                                        "be configured in this mode.\n");
1923                                 return;
1924                         }
1925                 }
1926                 if (nb_fwd_lcores == 1) {
1927                         printf("In DCB mode,the nb forwarding cores "
1928                                "should be larger than 1.\n");
1929                         return;
1930                 }
1931         }
1932         test_done = 0;
1933
1934         fwd_config_setup();
1935
1936         if(!no_flush_rx)
1937                 flush_fwd_rx_queues();
1938
1939         pkt_fwd_config_display(&cur_fwd_config);
1940         rxtx_config_display();
1941
1942         fwd_stats_reset();
1943         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1944                 pt_id = fwd_ports_ids[i];
1945                 port = &ports[pt_id];
1946                 map_port_queue_stats_mapping_registers(pt_id, port);
1947         }
1948         if (with_tx_first) {
1949                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1950                 if (port_fwd_begin != NULL) {
1951                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1952                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1953                 }
1954                 while (with_tx_first--) {
1955                         launch_packet_forwarding(
1956                                         run_one_txonly_burst_on_core);
1957                         rte_eal_mp_wait_lcore();
1958                 }
1959                 port_fwd_end = tx_only_engine.port_fwd_end;
1960                 if (port_fwd_end != NULL) {
1961                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1962                                 (*port_fwd_end)(fwd_ports_ids[i]);
1963                 }
1964         }
1965         launch_packet_forwarding(start_pkt_forward_on_core);
1966 }
1967
1968 void
1969 stop_packet_forwarding(void)
1970 {
1971         port_fwd_end_t port_fwd_end;
1972         lcoreid_t lc_id;
1973         portid_t pt_id;
1974         int i;
1975
1976         if (test_done) {
1977                 printf("Packet forwarding not started\n");
1978                 return;
1979         }
1980         printf("Telling cores to stop...");
1981         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1982                 fwd_lcores[lc_id]->stopped = 1;
1983         printf("\nWaiting for lcores to finish...\n");
1984         rte_eal_mp_wait_lcore();
1985         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1986         if (port_fwd_end != NULL) {
1987                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1988                         pt_id = fwd_ports_ids[i];
1989                         (*port_fwd_end)(pt_id);
1990                 }
1991         }
1992
1993         fwd_stats_display();
1994
1995         printf("\nDone.\n");
1996         test_done = 1;
1997 }
1998
1999 void
2000 dev_set_link_up(portid_t pid)
2001 {
2002         if (rte_eth_dev_set_link_up(pid) < 0)
2003                 printf("\nSet link up fail.\n");
2004 }
2005
2006 void
2007 dev_set_link_down(portid_t pid)
2008 {
2009         if (rte_eth_dev_set_link_down(pid) < 0)
2010                 printf("\nSet link down fail.\n");
2011 }
2012
2013 static int
2014 all_ports_started(void)
2015 {
2016         portid_t pi;
2017         struct rte_port *port;
2018
2019         RTE_ETH_FOREACH_DEV(pi) {
2020                 port = &ports[pi];
2021                 /* Check if there is a port which is not started */
2022                 if ((port->port_status != RTE_PORT_STARTED) &&
2023                         (port->slave_flag == 0))
2024                         return 0;
2025         }
2026
2027         /* No port is not started */
2028         return 1;
2029 }
2030
2031 int
2032 port_is_stopped(portid_t port_id)
2033 {
2034         struct rte_port *port = &ports[port_id];
2035
2036         if ((port->port_status != RTE_PORT_STOPPED) &&
2037             (port->slave_flag == 0))
2038                 return 0;
2039         return 1;
2040 }
2041
2042 int
2043 all_ports_stopped(void)
2044 {
2045         portid_t pi;
2046
2047         RTE_ETH_FOREACH_DEV(pi) {
2048                 if (!port_is_stopped(pi))
2049                         return 0;
2050         }
2051
2052         return 1;
2053 }
2054
2055 int
2056 port_is_started(portid_t port_id)
2057 {
2058         if (port_id_is_invalid(port_id, ENABLED_WARN))
2059                 return 0;
2060
2061         if (ports[port_id].port_status != RTE_PORT_STARTED)
2062                 return 0;
2063
2064         return 1;
2065 }
2066
2067 /* Configure the Rx and Tx hairpin queues for the selected port. */
2068 static int
2069 setup_hairpin_queues(portid_t pi)
2070 {
2071         queueid_t qi;
2072         struct rte_eth_hairpin_conf hairpin_conf = {
2073                 .peer_count = 1,
2074         };
2075         int i;
2076         int diag;
2077         struct rte_port *port = &ports[pi];
2078
2079         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2080                 hairpin_conf.peers[0].port = pi;
2081                 hairpin_conf.peers[0].queue = i + nb_rxq;
2082                 diag = rte_eth_tx_hairpin_queue_setup
2083                         (pi, qi, nb_txd, &hairpin_conf);
2084                 i++;
2085                 if (diag == 0)
2086                         continue;
2087
2088                 /* Fail to setup rx queue, return */
2089                 if (rte_atomic16_cmpset(&(port->port_status),
2090                                         RTE_PORT_HANDLING,
2091                                         RTE_PORT_STOPPED) == 0)
2092                         printf("Port %d can not be set back "
2093                                         "to stopped\n", pi);
2094                 printf("Fail to configure port %d hairpin "
2095                                 "queues\n", pi);
2096                 /* try to reconfigure queues next time */
2097                 port->need_reconfig_queues = 1;
2098                 return -1;
2099         }
2100         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2101                 hairpin_conf.peers[0].port = pi;
2102                 hairpin_conf.peers[0].queue = i + nb_txq;
2103                 diag = rte_eth_rx_hairpin_queue_setup
2104                         (pi, qi, nb_rxd, &hairpin_conf);
2105                 i++;
2106                 if (diag == 0)
2107                         continue;
2108
2109                 /* Fail to setup rx queue, return */
2110                 if (rte_atomic16_cmpset(&(port->port_status),
2111                                         RTE_PORT_HANDLING,
2112                                         RTE_PORT_STOPPED) == 0)
2113                         printf("Port %d can not be set back "
2114                                         "to stopped\n", pi);
2115                 printf("Fail to configure port %d hairpin "
2116                                 "queues\n", pi);
2117                 /* try to reconfigure queues next time */
2118                 port->need_reconfig_queues = 1;
2119                 return -1;
2120         }
2121         return 0;
2122 }
2123
2124 int
2125 start_port(portid_t pid)
2126 {
2127         int diag, need_check_link_status = -1;
2128         portid_t pi;
2129         queueid_t qi;
2130         struct rte_port *port;
2131         struct rte_ether_addr mac_addr;
2132         struct rte_eth_hairpin_cap cap;
2133
2134         if (port_id_is_invalid(pid, ENABLED_WARN))
2135                 return 0;
2136
2137         if(dcb_config)
2138                 dcb_test = 1;
2139         RTE_ETH_FOREACH_DEV(pi) {
2140                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2141                         continue;
2142
2143                 need_check_link_status = 0;
2144                 port = &ports[pi];
2145                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2146                                                  RTE_PORT_HANDLING) == 0) {
2147                         printf("Port %d is now not stopped\n", pi);
2148                         continue;
2149                 }
2150
2151                 if (port->need_reconfig > 0) {
2152                         port->need_reconfig = 0;
2153
2154                         if (flow_isolate_all) {
2155                                 int ret = port_flow_isolate(pi, 1);
2156                                 if (ret) {
2157                                         printf("Failed to apply isolated"
2158                                                " mode on port %d\n", pi);
2159                                         return -1;
2160                                 }
2161                         }
2162                         configure_rxtx_dump_callbacks(0);
2163                         printf("Configuring Port %d (socket %u)\n", pi,
2164                                         port->socket_id);
2165                         if (nb_hairpinq > 0 &&
2166                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2167                                 printf("Port %d doesn't support hairpin "
2168                                        "queues\n", pi);
2169                                 return -1;
2170                         }
2171                         /* configure port */
2172                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2173                                                      nb_txq + nb_hairpinq,
2174                                                      &(port->dev_conf));
2175                         if (diag != 0) {
2176                                 if (rte_atomic16_cmpset(&(port->port_status),
2177                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2178                                         printf("Port %d can not be set back "
2179                                                         "to stopped\n", pi);
2180                                 printf("Fail to configure port %d\n", pi);
2181                                 /* try to reconfigure port next time */
2182                                 port->need_reconfig = 1;
2183                                 return -1;
2184                         }
2185                 }
2186                 if (port->need_reconfig_queues > 0) {
2187                         port->need_reconfig_queues = 0;
2188                         /* setup tx queues */
2189                         for (qi = 0; qi < nb_txq; qi++) {
2190                                 if ((numa_support) &&
2191                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2192                                         diag = rte_eth_tx_queue_setup(pi, qi,
2193                                                 port->nb_tx_desc[qi],
2194                                                 txring_numa[pi],
2195                                                 &(port->tx_conf[qi]));
2196                                 else
2197                                         diag = rte_eth_tx_queue_setup(pi, qi,
2198                                                 port->nb_tx_desc[qi],
2199                                                 port->socket_id,
2200                                                 &(port->tx_conf[qi]));
2201
2202                                 if (diag == 0)
2203                                         continue;
2204
2205                                 /* Fail to setup tx queue, return */
2206                                 if (rte_atomic16_cmpset(&(port->port_status),
2207                                                         RTE_PORT_HANDLING,
2208                                                         RTE_PORT_STOPPED) == 0)
2209                                         printf("Port %d can not be set back "
2210                                                         "to stopped\n", pi);
2211                                 printf("Fail to configure port %d tx queues\n",
2212                                        pi);
2213                                 /* try to reconfigure queues next time */
2214                                 port->need_reconfig_queues = 1;
2215                                 return -1;
2216                         }
2217                         for (qi = 0; qi < nb_rxq; qi++) {
2218                                 /* setup rx queues */
2219                                 if ((numa_support) &&
2220                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2221                                         struct rte_mempool * mp =
2222                                                 mbuf_pool_find(rxring_numa[pi]);
2223                                         if (mp == NULL) {
2224                                                 printf("Failed to setup RX queue:"
2225                                                         "No mempool allocation"
2226                                                         " on the socket %d\n",
2227                                                         rxring_numa[pi]);
2228                                                 return -1;
2229                                         }
2230
2231                                         diag = rte_eth_rx_queue_setup(pi, qi,
2232                                              port->nb_rx_desc[qi],
2233                                              rxring_numa[pi],
2234                                              &(port->rx_conf[qi]),
2235                                              mp);
2236                                 } else {
2237                                         struct rte_mempool *mp =
2238                                                 mbuf_pool_find(port->socket_id);
2239                                         if (mp == NULL) {
2240                                                 printf("Failed to setup RX queue:"
2241                                                         "No mempool allocation"
2242                                                         " on the socket %d\n",
2243                                                         port->socket_id);
2244                                                 return -1;
2245                                         }
2246                                         diag = rte_eth_rx_queue_setup(pi, qi,
2247                                              port->nb_rx_desc[qi],
2248                                              port->socket_id,
2249                                              &(port->rx_conf[qi]),
2250                                              mp);
2251                                 }
2252                                 if (diag == 0)
2253                                         continue;
2254
2255                                 /* Fail to setup rx queue, return */
2256                                 if (rte_atomic16_cmpset(&(port->port_status),
2257                                                         RTE_PORT_HANDLING,
2258                                                         RTE_PORT_STOPPED) == 0)
2259                                         printf("Port %d can not be set back "
2260                                                         "to stopped\n", pi);
2261                                 printf("Fail to configure port %d rx queues\n",
2262                                        pi);
2263                                 /* try to reconfigure queues next time */
2264                                 port->need_reconfig_queues = 1;
2265                                 return -1;
2266                         }
2267                         /* setup hairpin queues */
2268                         if (setup_hairpin_queues(pi) != 0)
2269                                 return -1;
2270                 }
2271                 configure_rxtx_dump_callbacks(verbose_level);
2272                 /* start port */
2273                 if (rte_eth_dev_start(pi) < 0) {
2274                         printf("Fail to start port %d\n", pi);
2275
2276                         /* Fail to setup rx queue, return */
2277                         if (rte_atomic16_cmpset(&(port->port_status),
2278                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2279                                 printf("Port %d can not be set back to "
2280                                                         "stopped\n", pi);
2281                         continue;
2282                 }
2283
2284                 if (rte_atomic16_cmpset(&(port->port_status),
2285                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2286                         printf("Port %d can not be set into started\n", pi);
2287
2288                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2289                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2290                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2291                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2292                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2293
2294                 /* at least one port started, need checking link status */
2295                 need_check_link_status = 1;
2296         }
2297
2298         if (need_check_link_status == 1 && !no_link_check)
2299                 check_all_ports_link_status(RTE_PORT_ALL);
2300         else if (need_check_link_status == 0)
2301                 printf("Please stop the ports first\n");
2302
2303         printf("Done\n");
2304         return 0;
2305 }
2306
2307 void
2308 stop_port(portid_t pid)
2309 {
2310         portid_t pi;
2311         struct rte_port *port;
2312         int need_check_link_status = 0;
2313
2314         if (dcb_test) {
2315                 dcb_test = 0;
2316                 dcb_config = 0;
2317         }
2318
2319         if (port_id_is_invalid(pid, ENABLED_WARN))
2320                 return;
2321
2322         printf("Stopping ports...\n");
2323
2324         RTE_ETH_FOREACH_DEV(pi) {
2325                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2326                         continue;
2327
2328                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2329                         printf("Please remove port %d from forwarding configuration.\n", pi);
2330                         continue;
2331                 }
2332
2333                 if (port_is_bonding_slave(pi)) {
2334                         printf("Please remove port %d from bonded device.\n", pi);
2335                         continue;
2336                 }
2337
2338                 port = &ports[pi];
2339                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2340                                                 RTE_PORT_HANDLING) == 0)
2341                         continue;
2342
2343                 rte_eth_dev_stop(pi);
2344
2345                 if (rte_atomic16_cmpset(&(port->port_status),
2346                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2347                         printf("Port %d can not be set into stopped\n", pi);
2348                 need_check_link_status = 1;
2349         }
2350         if (need_check_link_status && !no_link_check)
2351                 check_all_ports_link_status(RTE_PORT_ALL);
2352
2353         printf("Done\n");
2354 }
2355
2356 static void
2357 remove_invalid_ports_in(portid_t *array, portid_t *total)
2358 {
2359         portid_t i;
2360         portid_t new_total = 0;
2361
2362         for (i = 0; i < *total; i++)
2363                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2364                         array[new_total] = array[i];
2365                         new_total++;
2366                 }
2367         *total = new_total;
2368 }
2369
2370 static void
2371 remove_invalid_ports(void)
2372 {
2373         remove_invalid_ports_in(ports_ids, &nb_ports);
2374         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2375         nb_cfg_ports = nb_fwd_ports;
2376 }
2377
2378 void
2379 close_port(portid_t pid)
2380 {
2381         portid_t pi;
2382         struct rte_port *port;
2383
2384         if (port_id_is_invalid(pid, ENABLED_WARN))
2385                 return;
2386
2387         printf("Closing ports...\n");
2388
2389         RTE_ETH_FOREACH_DEV(pi) {
2390                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2391                         continue;
2392
2393                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2394                         printf("Please remove port %d from forwarding configuration.\n", pi);
2395                         continue;
2396                 }
2397
2398                 if (port_is_bonding_slave(pi)) {
2399                         printf("Please remove port %d from bonded device.\n", pi);
2400                         continue;
2401                 }
2402
2403                 port = &ports[pi];
2404                 if (rte_atomic16_cmpset(&(port->port_status),
2405                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2406                         printf("Port %d is already closed\n", pi);
2407                         continue;
2408                 }
2409
2410                 if (rte_atomic16_cmpset(&(port->port_status),
2411                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2412                         printf("Port %d is now not stopped\n", pi);
2413                         continue;
2414                 }
2415
2416                 if (port->flow_list)
2417                         port_flow_flush(pi);
2418                 rte_eth_dev_close(pi);
2419
2420                 remove_invalid_ports();
2421
2422                 if (rte_atomic16_cmpset(&(port->port_status),
2423                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2424                         printf("Port %d cannot be set to closed\n", pi);
2425         }
2426
2427         printf("Done\n");
2428 }
2429
2430 void
2431 reset_port(portid_t pid)
2432 {
2433         int diag;
2434         portid_t pi;
2435         struct rte_port *port;
2436
2437         if (port_id_is_invalid(pid, ENABLED_WARN))
2438                 return;
2439
2440         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2441                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2442                 printf("Can not reset port(s), please stop port(s) first.\n");
2443                 return;
2444         }
2445
2446         printf("Resetting ports...\n");
2447
2448         RTE_ETH_FOREACH_DEV(pi) {
2449                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2450                         continue;
2451
2452                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2453                         printf("Please remove port %d from forwarding "
2454                                "configuration.\n", pi);
2455                         continue;
2456                 }
2457
2458                 if (port_is_bonding_slave(pi)) {
2459                         printf("Please remove port %d from bonded device.\n",
2460                                pi);
2461                         continue;
2462                 }
2463
2464                 diag = rte_eth_dev_reset(pi);
2465                 if (diag == 0) {
2466                         port = &ports[pi];
2467                         port->need_reconfig = 1;
2468                         port->need_reconfig_queues = 1;
2469                 } else {
2470                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2471                 }
2472         }
2473
2474         printf("Done\n");
2475 }
2476
2477 void
2478 attach_port(char *identifier)
2479 {
2480         portid_t pi;
2481         struct rte_dev_iterator iterator;
2482
2483         printf("Attaching a new port...\n");
2484
2485         if (identifier == NULL) {
2486                 printf("Invalid parameters are specified\n");
2487                 return;
2488         }
2489
2490         if (rte_dev_probe(identifier) < 0) {
2491                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2492                 return;
2493         }
2494
2495         /* first attach mode: event */
2496         if (setup_on_probe_event) {
2497                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2498                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2499                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2500                                         ports[pi].need_setup != 0)
2501                                 setup_attached_port(pi);
2502                 return;
2503         }
2504
2505         /* second attach mode: iterator */
2506         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2507                 /* setup ports matching the devargs used for probing */
2508                 if (port_is_forwarding(pi))
2509                         continue; /* port was already attached before */
2510                 setup_attached_port(pi);
2511         }
2512 }
2513
2514 static void
2515 setup_attached_port(portid_t pi)
2516 {
2517         unsigned int socket_id;
2518         int ret;
2519
2520         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2521         /* if socket_id is invalid, set to the first available socket. */
2522         if (check_socket_id(socket_id) < 0)
2523                 socket_id = socket_ids[0];
2524         reconfig(pi, socket_id);
2525         ret = rte_eth_promiscuous_enable(pi);
2526         if (ret != 0)
2527                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2528                         pi, rte_strerror(-ret));
2529
2530         ports_ids[nb_ports++] = pi;
2531         fwd_ports_ids[nb_fwd_ports++] = pi;
2532         nb_cfg_ports = nb_fwd_ports;
2533         ports[pi].need_setup = 0;
2534         ports[pi].port_status = RTE_PORT_STOPPED;
2535
2536         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2537         printf("Done\n");
2538 }
2539
2540 void
2541 detach_port_device(portid_t port_id)
2542 {
2543         struct rte_device *dev;
2544         portid_t sibling;
2545
2546         printf("Removing a device...\n");
2547
2548         dev = rte_eth_devices[port_id].device;
2549         if (dev == NULL) {
2550                 printf("Device already removed\n");
2551                 return;
2552         }
2553
2554         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2555                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2556                         printf("Port not stopped\n");
2557                         return;
2558                 }
2559                 printf("Port was not closed\n");
2560                 if (ports[port_id].flow_list)
2561                         port_flow_flush(port_id);
2562         }
2563
2564         if (rte_dev_remove(dev) < 0) {
2565                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2566                 return;
2567         }
2568         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2569                 /* reset mapping between old ports and removed device */
2570                 rte_eth_devices[sibling].device = NULL;
2571                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2572                         /* sibling ports are forced to be closed */
2573                         ports[sibling].port_status = RTE_PORT_CLOSED;
2574                         printf("Port %u is closed\n", sibling);
2575                 }
2576         }
2577
2578         remove_invalid_ports();
2579
2580         printf("Device of port %u is detached\n", port_id);
2581         printf("Now total ports is %d\n", nb_ports);
2582         printf("Done\n");
2583         return;
2584 }
2585
2586 void
2587 detach_device(char *identifier)
2588 {
2589         struct rte_dev_iterator iterator;
2590         struct rte_devargs da;
2591         portid_t port_id;
2592
2593         printf("Removing a device...\n");
2594
2595         memset(&da, 0, sizeof(da));
2596         if (rte_devargs_parsef(&da, "%s", identifier)) {
2597                 printf("cannot parse identifier\n");
2598                 if (da.args)
2599                         free(da.args);
2600                 return;
2601         }
2602
2603         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2604                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2605                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2606                                 printf("Port %u not stopped\n", port_id);
2607                                 return;
2608                         }
2609
2610                         /* sibling ports are forced to be closed */
2611                         if (ports[port_id].flow_list)
2612                                 port_flow_flush(port_id);
2613                         ports[port_id].port_status = RTE_PORT_CLOSED;
2614                         printf("Port %u is now closed\n", port_id);
2615                 }
2616         }
2617
2618         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2619                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2620                             da.name, da.bus->name);
2621                 return;
2622         }
2623
2624         remove_invalid_ports();
2625
2626         printf("Device %s is detached\n", identifier);
2627         printf("Now total ports is %d\n", nb_ports);
2628         printf("Done\n");
2629 }
2630
2631 void
2632 pmd_test_exit(void)
2633 {
2634         portid_t pt_id;
2635         int ret;
2636         int i;
2637
2638         if (test_done == 0)
2639                 stop_packet_forwarding();
2640
2641         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2642                 if (mempools[i]) {
2643                         if (mp_alloc_type == MP_ALLOC_ANON)
2644                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2645                                                      NULL);
2646                 }
2647         }
2648         if (ports != NULL) {
2649                 no_link_check = 1;
2650                 RTE_ETH_FOREACH_DEV(pt_id) {
2651                         printf("\nStopping port %d...\n", pt_id);
2652                         fflush(stdout);
2653                         stop_port(pt_id);
2654                 }
2655                 RTE_ETH_FOREACH_DEV(pt_id) {
2656                         printf("\nShutting down port %d...\n", pt_id);
2657                         fflush(stdout);
2658                         close_port(pt_id);
2659                 }
2660         }
2661
2662         if (hot_plug) {
2663                 ret = rte_dev_event_monitor_stop();
2664                 if (ret) {
2665                         RTE_LOG(ERR, EAL,
2666                                 "fail to stop device event monitor.");
2667                         return;
2668                 }
2669
2670                 ret = rte_dev_event_callback_unregister(NULL,
2671                         dev_event_callback, NULL);
2672                 if (ret < 0) {
2673                         RTE_LOG(ERR, EAL,
2674                                 "fail to unregister device event callback.\n");
2675                         return;
2676                 }
2677
2678                 ret = rte_dev_hotplug_handle_disable();
2679                 if (ret) {
2680                         RTE_LOG(ERR, EAL,
2681                                 "fail to disable hotplug handling.\n");
2682                         return;
2683                 }
2684         }
2685         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2686                 if (mempools[i])
2687                         rte_mempool_free(mempools[i]);
2688         }
2689
2690         printf("\nBye...\n");
2691 }
2692
2693 typedef void (*cmd_func_t)(void);
2694 struct pmd_test_command {
2695         const char *cmd_name;
2696         cmd_func_t cmd_func;
2697 };
2698
2699 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2700
2701 /* Check the link status of all ports in up to 9s, and print them finally */
2702 static void
2703 check_all_ports_link_status(uint32_t port_mask)
2704 {
2705 #define CHECK_INTERVAL 100 /* 100ms */
2706 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2707         portid_t portid;
2708         uint8_t count, all_ports_up, print_flag = 0;
2709         struct rte_eth_link link;
2710         int ret;
2711
2712         printf("Checking link statuses...\n");
2713         fflush(stdout);
2714         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2715                 all_ports_up = 1;
2716                 RTE_ETH_FOREACH_DEV(portid) {
2717                         if ((port_mask & (1 << portid)) == 0)
2718                                 continue;
2719                         memset(&link, 0, sizeof(link));
2720                         ret = rte_eth_link_get_nowait(portid, &link);
2721                         if (ret < 0) {
2722                                 all_ports_up = 0;
2723                                 if (print_flag == 1)
2724                                         printf("Port %u link get failed: %s\n",
2725                                                 portid, rte_strerror(-ret));
2726                                 continue;
2727                         }
2728                         /* print link status if flag set */
2729                         if (print_flag == 1) {
2730                                 if (link.link_status)
2731                                         printf(
2732                                         "Port%d Link Up. speed %u Mbps- %s\n",
2733                                         portid, link.link_speed,
2734                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2735                                         ("full-duplex") : ("half-duplex\n"));
2736                                 else
2737                                         printf("Port %d Link Down\n", portid);
2738                                 continue;
2739                         }
2740                         /* clear all_ports_up flag if any link down */
2741                         if (link.link_status == ETH_LINK_DOWN) {
2742                                 all_ports_up = 0;
2743                                 break;
2744                         }
2745                 }
2746                 /* after finally printing all link status, get out */
2747                 if (print_flag == 1)
2748                         break;
2749
2750                 if (all_ports_up == 0) {
2751                         fflush(stdout);
2752                         rte_delay_ms(CHECK_INTERVAL);
2753                 }
2754
2755                 /* set the print_flag if all ports up or timeout */
2756                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2757                         print_flag = 1;
2758                 }
2759
2760                 if (lsc_interrupt)
2761                         break;
2762         }
2763 }
2764
2765 /*
2766  * This callback is for remove a port for a device. It has limitation because
2767  * it is not for multiple port removal for a device.
2768  * TODO: the device detach invoke will plan to be removed from user side to
2769  * eal. And convert all PMDs to free port resources on ether device closing.
2770  */
2771 static void
2772 rmv_port_callback(void *arg)
2773 {
2774         int need_to_start = 0;
2775         int org_no_link_check = no_link_check;
2776         portid_t port_id = (intptr_t)arg;
2777
2778         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2779
2780         if (!test_done && port_is_forwarding(port_id)) {
2781                 need_to_start = 1;
2782                 stop_packet_forwarding();
2783         }
2784         no_link_check = 1;
2785         stop_port(port_id);
2786         no_link_check = org_no_link_check;
2787         close_port(port_id);
2788         detach_port_device(port_id);
2789         if (need_to_start)
2790                 start_packet_forwarding(0);
2791 }
2792
2793 /* This function is used by the interrupt thread */
2794 static int
2795 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2796                   void *ret_param)
2797 {
2798         RTE_SET_USED(param);
2799         RTE_SET_USED(ret_param);
2800
2801         if (type >= RTE_ETH_EVENT_MAX) {
2802                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2803                         port_id, __func__, type);
2804                 fflush(stderr);
2805         } else if (event_print_mask & (UINT32_C(1) << type)) {
2806                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2807                         eth_event_desc[type]);
2808                 fflush(stdout);
2809         }
2810
2811         switch (type) {
2812         case RTE_ETH_EVENT_NEW:
2813                 ports[port_id].need_setup = 1;
2814                 ports[port_id].port_status = RTE_PORT_HANDLING;
2815                 break;
2816         case RTE_ETH_EVENT_INTR_RMV:
2817                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2818                         break;
2819                 if (rte_eal_alarm_set(100000,
2820                                 rmv_port_callback, (void *)(intptr_t)port_id))
2821                         fprintf(stderr, "Could not set up deferred device removal\n");
2822                 break;
2823         default:
2824                 break;
2825         }
2826         return 0;
2827 }
2828
2829 static int
2830 register_eth_event_callback(void)
2831 {
2832         int ret;
2833         enum rte_eth_event_type event;
2834
2835         for (event = RTE_ETH_EVENT_UNKNOWN;
2836                         event < RTE_ETH_EVENT_MAX; event++) {
2837                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2838                                 event,
2839                                 eth_event_callback,
2840                                 NULL);
2841                 if (ret != 0) {
2842                         TESTPMD_LOG(ERR, "Failed to register callback for "
2843                                         "%s event\n", eth_event_desc[event]);
2844                         return -1;
2845                 }
2846         }
2847
2848         return 0;
2849 }
2850
2851 /* This function is used by the interrupt thread */
2852 static void
2853 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2854                              __rte_unused void *arg)
2855 {
2856         uint16_t port_id;
2857         int ret;
2858
2859         if (type >= RTE_DEV_EVENT_MAX) {
2860                 fprintf(stderr, "%s called upon invalid event %d\n",
2861                         __func__, type);
2862                 fflush(stderr);
2863         }
2864
2865         switch (type) {
2866         case RTE_DEV_EVENT_REMOVE:
2867                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2868                         device_name);
2869                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2870                 if (ret) {
2871                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2872                                 device_name);
2873                         return;
2874                 }
2875                 /*
2876                  * Because the user's callback is invoked in eal interrupt
2877                  * callback, the interrupt callback need to be finished before
2878                  * it can be unregistered when detaching device. So finish
2879                  * callback soon and use a deferred removal to detach device
2880                  * is need. It is a workaround, once the device detaching be
2881                  * moved into the eal in the future, the deferred removal could
2882                  * be deleted.
2883                  */
2884                 if (rte_eal_alarm_set(100000,
2885                                 rmv_port_callback, (void *)(intptr_t)port_id))
2886                         RTE_LOG(ERR, EAL,
2887                                 "Could not set up deferred device removal\n");
2888                 break;
2889         case RTE_DEV_EVENT_ADD:
2890                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2891                         device_name);
2892                 /* TODO: After finish kernel driver binding,
2893                  * begin to attach port.
2894                  */
2895                 break;
2896         default:
2897                 break;
2898         }
2899 }
2900
2901 static int
2902 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2903 {
2904         uint16_t i;
2905         int diag;
2906         uint8_t mapping_found = 0;
2907
2908         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2909                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2910                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2911                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2912                                         tx_queue_stats_mappings[i].queue_id,
2913                                         tx_queue_stats_mappings[i].stats_counter_id);
2914                         if (diag != 0)
2915                                 return diag;
2916                         mapping_found = 1;
2917                 }
2918         }
2919         if (mapping_found)
2920                 port->tx_queue_stats_mapping_enabled = 1;
2921         return 0;
2922 }
2923
2924 static int
2925 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2926 {
2927         uint16_t i;
2928         int diag;
2929         uint8_t mapping_found = 0;
2930
2931         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2932                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2933                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2934                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2935                                         rx_queue_stats_mappings[i].queue_id,
2936                                         rx_queue_stats_mappings[i].stats_counter_id);
2937                         if (diag != 0)
2938                                 return diag;
2939                         mapping_found = 1;
2940                 }
2941         }
2942         if (mapping_found)
2943                 port->rx_queue_stats_mapping_enabled = 1;
2944         return 0;
2945 }
2946
2947 static void
2948 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2949 {
2950         int diag = 0;
2951
2952         diag = set_tx_queue_stats_mapping_registers(pi, port);
2953         if (diag != 0) {
2954                 if (diag == -ENOTSUP) {
2955                         port->tx_queue_stats_mapping_enabled = 0;
2956                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2957                 }
2958                 else
2959                         rte_exit(EXIT_FAILURE,
2960                                         "set_tx_queue_stats_mapping_registers "
2961                                         "failed for port id=%d diag=%d\n",
2962                                         pi, diag);
2963         }
2964
2965         diag = set_rx_queue_stats_mapping_registers(pi, port);
2966         if (diag != 0) {
2967                 if (diag == -ENOTSUP) {
2968                         port->rx_queue_stats_mapping_enabled = 0;
2969                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2970                 }
2971                 else
2972                         rte_exit(EXIT_FAILURE,
2973                                         "set_rx_queue_stats_mapping_registers "
2974                                         "failed for port id=%d diag=%d\n",
2975                                         pi, diag);
2976         }
2977 }
2978
2979 static void
2980 rxtx_port_config(struct rte_port *port)
2981 {
2982         uint16_t qid;
2983         uint64_t offloads;
2984
2985         for (qid = 0; qid < nb_rxq; qid++) {
2986                 offloads = port->rx_conf[qid].offloads;
2987                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2988                 if (offloads != 0)
2989                         port->rx_conf[qid].offloads = offloads;
2990
2991                 /* Check if any Rx parameters have been passed */
2992                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2993                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2994
2995                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2996                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2997
2998                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2999                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3000
3001                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3002                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3003
3004                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3005                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3006
3007                 port->nb_rx_desc[qid] = nb_rxd;
3008         }
3009
3010         for (qid = 0; qid < nb_txq; qid++) {
3011                 offloads = port->tx_conf[qid].offloads;
3012                 port->tx_conf[qid] = port->dev_info.default_txconf;
3013                 if (offloads != 0)
3014                         port->tx_conf[qid].offloads = offloads;
3015
3016                 /* Check if any Tx parameters have been passed */
3017                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3018                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3019
3020                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3021                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3022
3023                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3024                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3025
3026                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3027                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3028
3029                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3030                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3031
3032                 port->nb_tx_desc[qid] = nb_txd;
3033         }
3034 }
3035
3036 void
3037 init_port_config(void)
3038 {
3039         portid_t pid;
3040         struct rte_port *port;
3041         int ret;
3042
3043         RTE_ETH_FOREACH_DEV(pid) {
3044                 port = &ports[pid];
3045                 port->dev_conf.fdir_conf = fdir_conf;
3046
3047                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3048                 if (ret != 0)
3049                         return;
3050
3051                 if (nb_rxq > 1) {
3052                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3053                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3054                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3055                 } else {
3056                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3057                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3058                 }
3059
3060                 if (port->dcb_flag == 0) {
3061                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3062                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
3063                         else
3064                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3065                 }
3066
3067                 rxtx_port_config(port);
3068
3069                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3070                 if (ret != 0)
3071                         return;
3072
3073                 map_port_queue_stats_mapping_registers(pid, port);
3074 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3075                 rte_pmd_ixgbe_bypass_init(pid);
3076 #endif
3077
3078                 if (lsc_interrupt &&
3079                     (rte_eth_devices[pid].data->dev_flags &
3080                      RTE_ETH_DEV_INTR_LSC))
3081                         port->dev_conf.intr_conf.lsc = 1;
3082                 if (rmv_interrupt &&
3083                     (rte_eth_devices[pid].data->dev_flags &
3084                      RTE_ETH_DEV_INTR_RMV))
3085                         port->dev_conf.intr_conf.rmv = 1;
3086         }
3087 }
3088
3089 void set_port_slave_flag(portid_t slave_pid)
3090 {
3091         struct rte_port *port;
3092
3093         port = &ports[slave_pid];
3094         port->slave_flag = 1;
3095 }
3096
3097 void clear_port_slave_flag(portid_t slave_pid)
3098 {
3099         struct rte_port *port;
3100
3101         port = &ports[slave_pid];
3102         port->slave_flag = 0;
3103 }
3104
3105 uint8_t port_is_bonding_slave(portid_t slave_pid)
3106 {
3107         struct rte_port *port;
3108
3109         port = &ports[slave_pid];
3110         if ((rte_eth_devices[slave_pid].data->dev_flags &
3111             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3112                 return 1;
3113         return 0;
3114 }
3115
3116 const uint16_t vlan_tags[] = {
3117                 0,  1,  2,  3,  4,  5,  6,  7,
3118                 8,  9, 10, 11,  12, 13, 14, 15,
3119                 16, 17, 18, 19, 20, 21, 22, 23,
3120                 24, 25, 26, 27, 28, 29, 30, 31
3121 };
3122
3123 static  int
3124 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3125                  enum dcb_mode_enable dcb_mode,
3126                  enum rte_eth_nb_tcs num_tcs,
3127                  uint8_t pfc_en)
3128 {
3129         uint8_t i;
3130         int32_t rc;
3131         struct rte_eth_rss_conf rss_conf;
3132
3133         /*
3134          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3135          * given above, and the number of traffic classes available for use.
3136          */
3137         if (dcb_mode == DCB_VT_ENABLED) {
3138                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3139                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3140                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3141                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3142
3143                 /* VMDQ+DCB RX and TX configurations */
3144                 vmdq_rx_conf->enable_default_pool = 0;
3145                 vmdq_rx_conf->default_pool = 0;
3146                 vmdq_rx_conf->nb_queue_pools =
3147                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3148                 vmdq_tx_conf->nb_queue_pools =
3149                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3150
3151                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3152                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3153                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3154                         vmdq_rx_conf->pool_map[i].pools =
3155                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3156                 }
3157                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3158                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3159                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3160                 }
3161
3162                 /* set DCB mode of RX and TX of multiple queues */
3163                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
3164                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3165         } else {
3166                 struct rte_eth_dcb_rx_conf *rx_conf =
3167                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3168                 struct rte_eth_dcb_tx_conf *tx_conf =
3169                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3170
3171                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3172                 if (rc != 0)
3173                         return rc;
3174
3175                 rx_conf->nb_tcs = num_tcs;
3176                 tx_conf->nb_tcs = num_tcs;
3177
3178                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3179                         rx_conf->dcb_tc[i] = i % num_tcs;
3180                         tx_conf->dcb_tc[i] = i % num_tcs;
3181                 }
3182
3183                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
3184                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3185                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3186         }
3187
3188         if (pfc_en)
3189                 eth_conf->dcb_capability_en =
3190                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3191         else
3192                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3193
3194         return 0;
3195 }
3196
3197 int
3198 init_port_dcb_config(portid_t pid,
3199                      enum dcb_mode_enable dcb_mode,
3200                      enum rte_eth_nb_tcs num_tcs,
3201                      uint8_t pfc_en)
3202 {
3203         struct rte_eth_conf port_conf;
3204         struct rte_port *rte_port;
3205         int retval;
3206         uint16_t i;
3207
3208         rte_port = &ports[pid];
3209
3210         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3211         /* Enter DCB configuration status */
3212         dcb_config = 1;
3213
3214         port_conf.rxmode = rte_port->dev_conf.rxmode;
3215         port_conf.txmode = rte_port->dev_conf.txmode;
3216
3217         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3218         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3219         if (retval < 0)
3220                 return retval;
3221         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3222
3223         /* re-configure the device . */
3224         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3225         if (retval < 0)
3226                 return retval;
3227
3228         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3229         if (retval != 0)
3230                 return retval;
3231
3232         /* If dev_info.vmdq_pool_base is greater than 0,
3233          * the queue id of vmdq pools is started after pf queues.
3234          */
3235         if (dcb_mode == DCB_VT_ENABLED &&
3236             rte_port->dev_info.vmdq_pool_base > 0) {
3237                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3238                         " for port %d.", pid);
3239                 return -1;
3240         }
3241
3242         /* Assume the ports in testpmd have the same dcb capability
3243          * and has the same number of rxq and txq in dcb mode
3244          */
3245         if (dcb_mode == DCB_VT_ENABLED) {
3246                 if (rte_port->dev_info.max_vfs > 0) {
3247                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3248                         nb_txq = rte_port->dev_info.nb_tx_queues;
3249                 } else {
3250                         nb_rxq = rte_port->dev_info.max_rx_queues;
3251                         nb_txq = rte_port->dev_info.max_tx_queues;
3252                 }
3253         } else {
3254                 /*if vt is disabled, use all pf queues */
3255                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3256                         nb_rxq = rte_port->dev_info.max_rx_queues;
3257                         nb_txq = rte_port->dev_info.max_tx_queues;
3258                 } else {
3259                         nb_rxq = (queueid_t)num_tcs;
3260                         nb_txq = (queueid_t)num_tcs;
3261
3262                 }
3263         }
3264         rx_free_thresh = 64;
3265
3266         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3267
3268         rxtx_port_config(rte_port);
3269         /* VLAN filter */
3270         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3271         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3272                 rx_vft_set(pid, vlan_tags[i], 1);
3273
3274         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3275         if (retval != 0)
3276                 return retval;
3277
3278         map_port_queue_stats_mapping_registers(pid, rte_port);
3279
3280         rte_port->dcb_flag = 1;
3281
3282         return 0;
3283 }
3284
3285 static void
3286 init_port(void)
3287 {
3288         /* Configuration of Ethernet ports. */
3289         ports = rte_zmalloc("testpmd: ports",
3290                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3291                             RTE_CACHE_LINE_SIZE);
3292         if (ports == NULL) {
3293                 rte_exit(EXIT_FAILURE,
3294                                 "rte_zmalloc(%d struct rte_port) failed\n",
3295                                 RTE_MAX_ETHPORTS);
3296         }
3297
3298         /* Initialize ports NUMA structures */
3299         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3300         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3301         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3302 }
3303
3304 static void
3305 force_quit(void)
3306 {
3307         pmd_test_exit();
3308         prompt_exit();
3309 }
3310
3311 static void
3312 print_stats(void)
3313 {
3314         uint8_t i;
3315         const char clr[] = { 27, '[', '2', 'J', '\0' };
3316         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3317
3318         /* Clear screen and move to top left */
3319         printf("%s%s", clr, top_left);
3320
3321         printf("\nPort statistics ====================================");
3322         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3323                 nic_stats_display(fwd_ports_ids[i]);
3324
3325         fflush(stdout);
3326 }
3327
3328 static void
3329 signal_handler(int signum)
3330 {
3331         if (signum == SIGINT || signum == SIGTERM) {
3332                 printf("\nSignal %d received, preparing to exit...\n",
3333                                 signum);
3334 #ifdef RTE_LIBRTE_PDUMP
3335                 /* uninitialize packet capture framework */
3336                 rte_pdump_uninit();
3337 #endif
3338 #ifdef RTE_LIBRTE_LATENCY_STATS
3339                 if (latencystats_enabled != 0)
3340                         rte_latencystats_uninit();
3341 #endif
3342                 force_quit();
3343                 /* Set flag to indicate the force termination. */
3344                 f_quit = 1;
3345                 /* exit with the expected status */
3346                 signal(signum, SIG_DFL);
3347                 kill(getpid(), signum);
3348         }
3349 }
3350
3351 int
3352 main(int argc, char** argv)
3353 {
3354         int diag;
3355         portid_t port_id;
3356         uint16_t count;
3357         int ret;
3358
3359         signal(SIGINT, signal_handler);
3360         signal(SIGTERM, signal_handler);
3361
3362         testpmd_logtype = rte_log_register("testpmd");
3363         if (testpmd_logtype < 0)
3364                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3365         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3366
3367         diag = rte_eal_init(argc, argv);
3368         if (diag < 0)
3369                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3370                          rte_strerror(rte_errno));
3371
3372         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3373                 rte_exit(EXIT_FAILURE,
3374                          "Secondary process type not supported.\n");
3375
3376         ret = register_eth_event_callback();
3377         if (ret != 0)
3378                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3379
3380 #ifdef RTE_LIBRTE_PDUMP
3381         /* initialize packet capture framework */
3382         rte_pdump_init();
3383 #endif
3384
3385         count = 0;
3386         RTE_ETH_FOREACH_DEV(port_id) {
3387                 ports_ids[count] = port_id;
3388                 count++;
3389         }
3390         nb_ports = (portid_t) count;
3391         if (nb_ports == 0)
3392                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3393
3394         /* allocate port structures, and init them */
3395         init_port();
3396
3397         set_def_fwd_config();
3398         if (nb_lcores == 0)
3399                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3400                          "Check the core mask argument\n");
3401
3402         /* Bitrate/latency stats disabled by default */
3403 #ifdef RTE_LIBRTE_BITRATE
3404         bitrate_enabled = 0;
3405 #endif
3406 #ifdef RTE_LIBRTE_LATENCY_STATS
3407         latencystats_enabled = 0;
3408 #endif
3409
3410         /* on FreeBSD, mlockall() is disabled by default */
3411 #ifdef RTE_EXEC_ENV_FREEBSD
3412         do_mlockall = 0;
3413 #else
3414         do_mlockall = 1;
3415 #endif
3416
3417         argc -= diag;
3418         argv += diag;
3419         if (argc > 1)
3420                 launch_args_parse(argc, argv);
3421
3422         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3423                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3424                         strerror(errno));
3425         }
3426
3427         if (tx_first && interactive)
3428                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3429                                 "interactive mode.\n");
3430
3431         if (tx_first && lsc_interrupt) {
3432                 printf("Warning: lsc_interrupt needs to be off when "
3433                                 " using tx_first. Disabling.\n");
3434                 lsc_interrupt = 0;
3435         }
3436
3437         if (!nb_rxq && !nb_txq)
3438                 printf("Warning: Either rx or tx queues should be non-zero\n");
3439
3440         if (nb_rxq > 1 && nb_rxq > nb_txq)
3441                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3442                        "but nb_txq=%d will prevent to fully test it.\n",
3443                        nb_rxq, nb_txq);
3444
3445         init_config();
3446
3447         if (hot_plug) {
3448                 ret = rte_dev_hotplug_handle_enable();
3449                 if (ret) {
3450                         RTE_LOG(ERR, EAL,
3451                                 "fail to enable hotplug handling.");
3452                         return -1;
3453                 }
3454
3455                 ret = rte_dev_event_monitor_start();
3456                 if (ret) {
3457                         RTE_LOG(ERR, EAL,
3458                                 "fail to start device event monitoring.");
3459                         return -1;
3460                 }
3461
3462                 ret = rte_dev_event_callback_register(NULL,
3463                         dev_event_callback, NULL);
3464                 if (ret) {
3465                         RTE_LOG(ERR, EAL,
3466                                 "fail  to register device event callback\n");
3467                         return -1;
3468                 }
3469         }
3470
3471         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3472                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3473
3474         /* set all ports to promiscuous mode by default */
3475         RTE_ETH_FOREACH_DEV(port_id) {
3476                 ret = rte_eth_promiscuous_enable(port_id);
3477                 if (ret != 0)
3478                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3479                                 port_id, rte_strerror(-ret));
3480         }
3481
3482         /* Init metrics library */
3483         rte_metrics_init(rte_socket_id());
3484
3485 #ifdef RTE_LIBRTE_LATENCY_STATS
3486         if (latencystats_enabled != 0) {
3487                 int ret = rte_latencystats_init(1, NULL);
3488                 if (ret)
3489                         printf("Warning: latencystats init()"
3490                                 " returned error %d\n", ret);
3491                 printf("Latencystats running on lcore %d\n",
3492                         latencystats_lcore_id);
3493         }
3494 #endif
3495
3496         /* Setup bitrate stats */
3497 #ifdef RTE_LIBRTE_BITRATE
3498         if (bitrate_enabled != 0) {
3499                 bitrate_data = rte_stats_bitrate_create();
3500                 if (bitrate_data == NULL)
3501                         rte_exit(EXIT_FAILURE,
3502                                 "Could not allocate bitrate data.\n");
3503                 rte_stats_bitrate_reg(bitrate_data);
3504         }
3505 #endif
3506
3507 #ifdef RTE_LIBRTE_CMDLINE
3508         if (strlen(cmdline_filename) != 0)
3509                 cmdline_read_from_file(cmdline_filename);
3510
3511         if (interactive == 1) {
3512                 if (auto_start) {
3513                         printf("Start automatic packet forwarding\n");
3514                         start_packet_forwarding(0);
3515                 }
3516                 prompt();
3517                 pmd_test_exit();
3518         } else
3519 #endif
3520         {
3521                 char c;
3522                 int rc;
3523
3524                 f_quit = 0;
3525
3526                 printf("No commandline core given, start packet forwarding\n");
3527                 start_packet_forwarding(tx_first);
3528                 if (stats_period != 0) {
3529                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3530                         uint64_t timer_period;
3531
3532                         /* Convert to number of cycles */
3533                         timer_period = stats_period * rte_get_timer_hz();
3534
3535                         while (f_quit == 0) {
3536                                 cur_time = rte_get_timer_cycles();
3537                                 diff_time += cur_time - prev_time;
3538
3539                                 if (diff_time >= timer_period) {
3540                                         print_stats();
3541                                         /* Reset the timer */
3542                                         diff_time = 0;
3543                                 }
3544                                 /* Sleep to avoid unnecessary checks */
3545                                 prev_time = cur_time;
3546                                 sleep(1);
3547                         }
3548                 }
3549
3550                 printf("Press enter to exit\n");
3551                 rc = read(0, &c, 1);
3552                 pmd_test_exit();
3553                 if (rc < 0)
3554                         return 1;
3555         }
3556
3557         return 0;
3558 }