app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATESTATS
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 200         DEFAULT_MBUF_DATA_SIZE
 201 }; /**< Mbuf data space size. */
 202 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 203                                       * specified on command-line. */
 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 205
 206 /*
 207  * In container, it cannot terminate the process which running with 'stats-period'
 208  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 209  */
 210 uint8_t f_quit;
 211
 212 /*
 213  * Configuration of packet segments used to scatter received packets
 214  * if some of split features is configured.
 215  */
 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 217 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 219 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 220
 221 /*
 222  * Configuration of packet segments used by the "txonly" processing engine.
 223  */
 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 226         TXONLY_DEF_PACKET_LEN,
 227 };
 228 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 229
 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 231 /**< Split policy for packets to TX. */
 232
 233 uint8_t txonly_multi_flow;
 234 /**< Whether multiple flows are generated in TXONLY mode. */
 235
 236 uint32_t tx_pkt_times_inter;
 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 238
 239 uint32_t tx_pkt_times_intra;
 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 241
 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 243 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 244
 245 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 246 uint8_t dcb_config = 0;
 247
 248 /* Whether the dcb is in testing status */
 249 uint8_t dcb_test = 0;
 250
 251 /*
 252  * Configurable number of RX/TX queues.
 253  */
 254 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 255 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 256 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 257
 258 /*
 259  * Configurable number of RX/TX ring descriptors.
 260  * Defaults are supplied by drivers via ethdev.
 261  */
 262 #define RTE_TEST_RX_DESC_DEFAULT 0
 263 #define RTE_TEST_TX_DESC_DEFAULT 0
 264 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 265 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 266
 267 #define RTE_PMD_PARAM_UNSET -1
 268 /*
 269  * Configurable values of RX and TX ring threshold registers.
 270  */
 271
 272 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 273 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 274 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 275
 276 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 277 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 278 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 279
 280 /*
 281  * Configurable value of RX free threshold.
 282  */
 283 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 284
 285 /*
 286  * Configurable value of RX drop enable.
 287  */
 288 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 289
 290 /*
 291  * Configurable value of TX free threshold.
 292  */
 293 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 294
 295 /*
 296  * Configurable value of TX RS bit threshold.
 297  */
 298 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 299
 300 /*
 301  * Configurable value of buffered packets before sending.
 302  */
 303 uint16_t noisy_tx_sw_bufsz;
 304
 305 /*
 306  * Configurable value of packet buffer timeout.
 307  */
 308 uint16_t noisy_tx_sw_buf_flush_time;
 309
 310 /*
 311  * Configurable value for size of VNF internal memory area
 312  * used for simulating noisy neighbour behaviour
 313  */
 314 uint64_t noisy_lkup_mem_sz;
 315
 316 /*
 317  * Configurable value of number of random writes done in
 318  * VNF simulation memory area.
 319  */
 320 uint64_t noisy_lkup_num_writes;
 321
 322 /*
 323  * Configurable value of number of random reads done in
 324  * VNF simulation memory area.
 325  */
 326 uint64_t noisy_lkup_num_reads;
 327
 328 /*
 329  * Configurable value of number of random reads/writes done in
 330  * VNF simulation memory area.
 331  */
 332 uint64_t noisy_lkup_num_reads_writes;
 333
 334 /*
 335  * Receive Side Scaling (RSS) configuration.
 336  */
 337 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 338
 339 /*
 340  * Port topology configuration
 341  */
 342 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 343
 344 /*
 345  * Avoids to flush all the RX streams before starts forwarding.
 346  */
 347 uint8_t no_flush_rx = 0; /* flush by default */
 348
 349 /*
 350  * Flow API isolated mode.
 351  */
 352 uint8_t flow_isolate_all;
 353
 354 /*
 355  * Avoids to check link status when starting/stopping a port.
 356  */
 357 uint8_t no_link_check = 0; /* check by default */
 358
 359 /*
 360  * Don't automatically start all ports in interactive mode.
 361  */
 362 uint8_t no_device_start = 0;
 363
 364 /*
 365  * Enable link status change notification
 366  */
 367 uint8_t lsc_interrupt = 1; /* enabled by default */
 368
 369 /*
 370  * Enable device removal notification.
 371  */
 372 uint8_t rmv_interrupt = 1; /* enabled by default */
 373
 374 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 375
 376 /* After attach, port setup is called on event or by iterator */
 377 bool setup_on_probe_event = true;
 378
 379 /* Clear ptypes on port initialization. */
 380 uint8_t clear_ptypes = true;
 381
 382 /* Hairpin ports configuration mode. */
 383 uint16_t hairpin_mode;
 384
 385 /* Pretty printing of ethdev events */
 386 static const char * const eth_event_desc[] = {
 387         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 388         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 389         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 390         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 391         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 392         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 393         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 394         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 395         [RTE_ETH_EVENT_NEW] = "device probed",
 396         [RTE_ETH_EVENT_DESTROY] = "device released",
 397         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 398         [RTE_ETH_EVENT_MAX] = NULL,
 399 };
 400
 401 /*
 402  * Display or mask ether events
 403  * Default to all events except VF_MBOX
 404  */
 405 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 406                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 407                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 408                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 413 /*
 414  * Decide if all memory are locked for performance.
 415  */
 416 int do_mlockall = 0;
 417
 418 /*
 419  * NIC bypass mode configuration options.
 420  */
 421
 422 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 423 /* The NIC bypass watchdog timeout. */
 424 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 425 #endif
 426
 427
 428 #ifdef RTE_LIBRTE_LATENCY_STATS
 429
 430 /*
 431  * Set when latency stats is enabled in the commandline
 432  */
 433 uint8_t latencystats_enabled;
 434
 435 /*
 436  * Lcore ID to serive latency statistics.
 437  */
 438 lcoreid_t latencystats_lcore_id = -1;
 439
 440 #endif
 441
 442 /*
 443  * Ethernet device configuration.
 444  */
 445 struct rte_eth_rxmode rx_mode = {
 446         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 447                 /**< Default maximum frame length. */
 448 };
 449
 450 struct rte_eth_txmode tx_mode = {
 451         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 452 };
 453
 454 struct rte_fdir_conf fdir_conf = {
 455         .mode = RTE_FDIR_MODE_NONE,
 456         .pballoc = RTE_FDIR_PBALLOC_64K,
 457         .status = RTE_FDIR_REPORT_STATUS,
 458         .mask = {
 459                 .vlan_tci_mask = 0xFFEF,
 460                 .ipv4_mask     = {
 461                         .src_ip = 0xFFFFFFFF,
 462                         .dst_ip = 0xFFFFFFFF,
 463                 },
 464                 .ipv6_mask     = {
 465                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 466                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 467                 },
 468                 .src_port_mask = 0xFFFF,
 469                 .dst_port_mask = 0xFFFF,
 470                 .mac_addr_byte_mask = 0xFF,
 471                 .tunnel_type_mask = 1,
 472                 .tunnel_id_mask = 0xFFFFFFFF,
 473         },
 474         .drop_queue = 127,
 475 };
 476
 477 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 478
 479 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 480 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 481
 482 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 483 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 484
 485 uint16_t nb_tx_queue_stats_mappings = 0;
 486 uint16_t nb_rx_queue_stats_mappings = 0;
 487
 488 /*
 489  * Display zero values by default for xstats
 490  */
 491 uint8_t xstats_hide_zero;
 492
 493 /*
 494  * Measure of CPU cycles disabled by default
 495  */
 496 uint8_t record_core_cycles;
 497
 498 /*
 499  * Display of RX and TX bursts disabled by default
 500  */
 501 uint8_t record_burst_stats;
 502
 503 unsigned int num_sockets = 0;
 504 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 505
 506 #ifdef RTE_LIBRTE_BITRATESTATS
 507 /* Bitrate statistics */
 508 struct rte_stats_bitrates *bitrate_data;
 509 lcoreid_t bitrate_lcore_id;
 510 uint8_t bitrate_enabled;
 511 #endif
 512
 513 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 514 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 515
 516 /*
 517  * hexadecimal bitmask of RX mq mode can be enabled.
 518  */
 519 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 520
 521 /* Forward function declarations */
 522 static void setup_attached_port(portid_t pi);
 523 static void map_port_queue_stats_mapping_registers(portid_t pi,
 524                                                    struct rte_port *port);
 525 static void check_all_ports_link_status(uint32_t port_mask);
 526 static int eth_event_callback(portid_t port_id,
 527                               enum rte_eth_event_type type,
 528                               void *param, void *ret_param);
 529 static void dev_event_callback(const char *device_name,
 530                                 enum rte_dev_event_type type,
 531                                 void *param);
 532
 533 /*
 534  * Check if all the ports are started.
 535  * If yes, return positive value. If not, return zero.
 536  */
 537 static int all_ports_started(void);
 538
 539 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 540 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 541
 542 /* Holds the registered mbuf dynamic flags names. */
 543 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 544
 545 /*
 546  * Helper function to check if socket is already discovered.
 547  * If yes, return positive value. If not, return zero.
 548  */
 549 int
 550 new_socket_id(unsigned int socket_id)
 551 {
 552         unsigned int i;
 553
 554         for (i = 0; i < num_sockets; i++) {
 555                 if (socket_ids[i] == socket_id)
 556                         return 0;
 557         }
 558         return 1;
 559 }
 560
 561 /*
 562  * Setup default configuration.
 563  */
 564 static void
 565 set_default_fwd_lcores_config(void)
 566 {
 567         unsigned int i;
 568         unsigned int nb_lc;
 569         unsigned int sock_num;
 570
 571         nb_lc = 0;
 572         for (i = 0; i < RTE_MAX_LCORE; i++) {
 573                 if (!rte_lcore_is_enabled(i))
 574                         continue;
 575                 sock_num = rte_lcore_to_socket_id(i);
 576                 if (new_socket_id(sock_num)) {
 577                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 578                                 rte_exit(EXIT_FAILURE,
 579                                          "Total sockets greater than %u\n",
 580                                          RTE_MAX_NUMA_NODES);
 581                         }
 582                         socket_ids[num_sockets++] = sock_num;
 583                 }
 584                 if (i == rte_get_master_lcore())
 585                         continue;
 586                 fwd_lcores_cpuids[nb_lc++] = i;
 587         }
 588         nb_lcores = (lcoreid_t) nb_lc;
 589         nb_cfg_lcores = nb_lcores;
 590         nb_fwd_lcores = 1;
 591 }
 592
 593 static void
 594 set_def_peer_eth_addrs(void)
 595 {
 596         portid_t i;
 597
 598         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 599                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 600                 peer_eth_addrs[i].addr_bytes[5] = i;
 601         }
 602 }
 603
 604 static void
 605 set_default_fwd_ports_config(void)
 606 {
 607         portid_t pt_id;
 608         int i = 0;
 609
 610         RTE_ETH_FOREACH_DEV(pt_id) {
 611                 fwd_ports_ids[i++] = pt_id;
 612
 613                 /* Update sockets info according to the attached device */
 614                 int socket_id = rte_eth_dev_socket_id(pt_id);
 615                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 616                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 617                                 rte_exit(EXIT_FAILURE,
 618                                          "Total sockets greater than %u\n",
 619                                          RTE_MAX_NUMA_NODES);
 620                         }
 621                         socket_ids[num_sockets++] = socket_id;
 622                 }
 623         }
 624
 625         nb_cfg_ports = nb_ports;
 626         nb_fwd_ports = nb_ports;
 627 }
 628
 629 void
 630 set_def_fwd_config(void)
 631 {
 632         set_default_fwd_lcores_config();
 633         set_def_peer_eth_addrs();
 634         set_default_fwd_ports_config();
 635 }
 636
 637 /* extremely pessimistic estimation of memory required to create a mempool */
 638 static int
 639 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 640 {
 641         unsigned int n_pages, mbuf_per_pg, leftover;
 642         uint64_t total_mem, mbuf_mem, obj_sz;
 643
 644         /* there is no good way to predict how much space the mempool will
 645          * occupy because it will allocate chunks on the fly, and some of those
 646          * will come from default DPDK memory while some will come from our
 647          * external memory, so just assume 128MB will be enough for everyone.
 648          */
 649         uint64_t hdr_mem = 128 << 20;
 650
 651         /* account for possible non-contiguousness */
 652         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 653         if (obj_sz > pgsz) {
 654                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 655                 return -1;
 656         }
 657
 658         mbuf_per_pg = pgsz / obj_sz;
 659         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 660         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 661
 662         mbuf_mem = n_pages * pgsz;
 663
 664         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 665
 666         if (total_mem > SIZE_MAX) {
 667                 TESTPMD_LOG(ERR, "Memory size too big\n");
 668                 return -1;
 669         }
 670         *out = (size_t)total_mem;
 671
 672         return 0;
 673 }
 674
 675 static int
 676 pagesz_flags(uint64_t page_sz)
 677 {
 678         /* as per mmap() manpage, all page sizes are log2 of page size
 679          * shifted by MAP_HUGE_SHIFT
 680          */
 681         int log2 = rte_log2_u64(page_sz);
 682
 683         return (log2 << HUGE_SHIFT);
 684 }
 685
 686 static void *
 687 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 688 {
 689         void *addr;
 690         int flags;
 691
 692         /* allocate anonymous hugepages */
 693         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 694         if (huge)
 695                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 696
 697         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 698         if (addr == MAP_FAILED)
 699                 return NULL;
 700
 701         return addr;
 702 }
 703
 704 struct extmem_param {
 705         void *addr;
 706         size_t len;
 707         size_t pgsz;
 708         rte_iova_t *iova_table;
 709         unsigned int iova_table_len;
 710 };
 711
 712 static int
 713 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 714                 bool huge)
 715 {
 716         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 717                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 718         unsigned int cur_page, n_pages, pgsz_idx;
 719         size_t mem_sz, cur_pgsz;
 720         rte_iova_t *iovas = NULL;
 721         void *addr;
 722         int ret;
 723
 724         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 725                 /* skip anything that is too big */
 726                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 727                         continue;
 728
 729                 cur_pgsz = pgsizes[pgsz_idx];
 730
 731                 /* if we were told not to allocate hugepages, override */
 732                 if (!huge)
 733                         cur_pgsz = sysconf(_SC_PAGESIZE);
 734
 735                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 736                 if (ret < 0) {
 737                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 738                         return -1;
 739                 }
 740
 741                 /* allocate our memory */
 742                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 743
 744                 /* if we couldn't allocate memory with a specified page size,
 745                  * that doesn't mean we can't do it with other page sizes, so
 746                  * try another one.
 747                  */
 748                 if (addr == NULL)
 749                         continue;
 750
 751                 /* store IOVA addresses for every page in this memory area */
 752                 n_pages = mem_sz / cur_pgsz;
 753
 754                 iovas = malloc(sizeof(*iovas) * n_pages);
 755
 756                 if (iovas == NULL) {
 757                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 758                         goto fail;
 759                 }
 760                 /* lock memory if it's not huge pages */
 761                 if (!huge)
 762                         mlock(addr, mem_sz);
 763
 764                 /* populate IOVA addresses */
 765                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 766                         rte_iova_t iova;
 767                         size_t offset;
 768                         void *cur;
 769
 770                         offset = cur_pgsz * cur_page;
 771                         cur = RTE_PTR_ADD(addr, offset);
 772
 773                         /* touch the page before getting its IOVA */
 774                         *(volatile char *)cur = 0;
 775
 776                         iova = rte_mem_virt2iova(cur);
 777
 778                         iovas[cur_page] = iova;
 779                 }
 780
 781                 break;
 782         }
 783         /* if we couldn't allocate anything */
 784         if (iovas == NULL)
 785                 return -1;
 786
 787         param->addr = addr;
 788         param->len = mem_sz;
 789         param->pgsz = cur_pgsz;
 790         param->iova_table = iovas;
 791         param->iova_table_len = n_pages;
 792
 793         return 0;
 794 fail:
 795         if (iovas)
 796                 free(iovas);
 797         if (addr)
 798                 munmap(addr, mem_sz);
 799
 800         return -1;
 801 }
 802
 803 static int
 804 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 805 {
 806         struct extmem_param param;
 807         int socket_id, ret;
 808
 809         memset(&param, 0, sizeof(param));
 810
 811         /* check if our heap exists */
 812         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 813         if (socket_id < 0) {
 814                 /* create our heap */
 815                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 816                 if (ret < 0) {
 817                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 818                         return -1;
 819                 }
 820         }
 821
 822         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 823         if (ret < 0) {
 824                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 825                 return -1;
 826         }
 827
 828         /* we now have a valid memory area, so add it to heap */
 829         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 830                         param.addr, param.len, param.iova_table,
 831                         param.iova_table_len, param.pgsz);
 832
 833         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 834
 835         /* not needed any more */
 836         free(param.iova_table);
 837
 838         if (ret < 0) {
 839                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 840                 munmap(param.addr, param.len);
 841                 return -1;
 842         }
 843
 844         /* success */
 845
 846         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 847                         param.len >> 20);
 848
 849         return 0;
 850 }
 851 static void
 852 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 853              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 854 {
 855         uint16_t pid = 0;
 856         int ret;
 857
 858         RTE_ETH_FOREACH_DEV(pid) {
 859                 struct rte_eth_dev *dev =
 860                         &rte_eth_devices[pid];
 861
 862                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 863                                         memhdr->len);
 864                 if (ret) {
 865                         TESTPMD_LOG(DEBUG,
 866                                     "unable to DMA unmap addr 0x%p "
 867                                     "for device %s\n",
 868                                     memhdr->addr, dev->data->name);
 869                 }
 870         }
 871         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 872         if (ret) {
 873                 TESTPMD_LOG(DEBUG,
 874                             "unable to un-register addr 0x%p\n", memhdr->addr);
 875         }
 876 }
 877
 878 static void
 879 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 880            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 881 {
 882         uint16_t pid = 0;
 883         size_t page_size = sysconf(_SC_PAGESIZE);
 884         int ret;
 885
 886         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 887                                   page_size);
 888         if (ret) {
 889                 TESTPMD_LOG(DEBUG,
 890                             "unable to register addr 0x%p\n", memhdr->addr);
 891                 return;
 892         }
 893         RTE_ETH_FOREACH_DEV(pid) {
 894                 struct rte_eth_dev *dev =
 895                         &rte_eth_devices[pid];
 896
 897                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 898                                       memhdr->len);
 899                 if (ret) {
 900                         TESTPMD_LOG(DEBUG,
 901                                     "unable to DMA map addr 0x%p "
 902                                     "for device %s\n",
 903                                     memhdr->addr, dev->data->name);
 904                 }
 905         }
 906 }
 907
 908 static unsigned int
 909 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 910             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 911 {
 912         struct rte_pktmbuf_extmem *xmem;
 913         unsigned int ext_num, zone_num, elt_num;
 914         uint16_t elt_size;
 915
 916         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 917         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 918         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 919
 920         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 921         if (xmem == NULL) {
 922                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 923                                  "external buffer descriptors\n");
 924                 *ext_mem = NULL;
 925                 return 0;
 926         }
 927         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 928                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 929                 const struct rte_memzone *mz;
 930                 char mz_name[RTE_MEMZONE_NAMESIZE];
 931                 int ret;
 932
 933                 ret = snprintf(mz_name, sizeof(mz_name),
 934                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 935                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 936                         errno = ENAMETOOLONG;
 937                         ext_num = 0;
 938                         break;
 939                 }
 940                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 941                                                  socket_id,
 942                                                  RTE_MEMZONE_IOVA_CONTIG |
 943                                                  RTE_MEMZONE_1GB |
 944                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 945                                                  EXTBUF_ZONE_SIZE);
 946                 if (mz == NULL) {
 947                         /*
 948                          * The caller exits on external buffer creation
 949                          * error, so there is no need to free memzones.
 950                          */
 951                         errno = ENOMEM;
 952                         ext_num = 0;
 953                         break;
 954                 }
 955                 xseg->buf_ptr = mz->addr;
 956                 xseg->buf_iova = mz->iova;
 957                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 958                 xseg->elt_size = elt_size;
 959         }
 960         if (ext_num == 0 && xmem != NULL) {
 961                 free(xmem);
 962                 xmem = NULL;
 963         }
 964         *ext_mem = xmem;
 965         return ext_num;
 966 }
 967
 968 /*
 969  * Configuration initialisation done once at init time.
 970  */
 971 static struct rte_mempool *
 972 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 973                  unsigned int socket_id, uint16_t size_idx)
 974 {
 975         char pool_name[RTE_MEMPOOL_NAMESIZE];
 976         struct rte_mempool *rte_mp = NULL;
 977         uint32_t mb_size;
 978
 979         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 980         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 981
 982         TESTPMD_LOG(INFO,
 983                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 984                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 985
 986         switch (mp_alloc_type) {
 987         case MP_ALLOC_NATIVE:
 988                 {
 989                         /* wrapper to rte_mempool_create() */
 990                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 991                                         rte_mbuf_best_mempool_ops());
 992                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 993                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 994                         break;
 995                 }
 996         case MP_ALLOC_ANON:
 997                 {
 998                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 999                                 mb_size, (unsigned int) mb_mempool_cache,
1000                                 sizeof(struct rte_pktmbuf_pool_private),
1001                                 socket_id, mempool_flags);
1002                         if (rte_mp == NULL)
1003                                 goto err;
1004
1005                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1006                                 rte_mempool_free(rte_mp);
1007                                 rte_mp = NULL;
1008                                 goto err;
1009                         }
1010                         rte_pktmbuf_pool_init(rte_mp, NULL);
1011                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1012                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1013                         break;
1014                 }
1015         case MP_ALLOC_XMEM:
1016         case MP_ALLOC_XMEM_HUGE:
1017                 {
1018                         int heap_socket;
1019                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1020
1021                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1022                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1023
1024                         heap_socket =
1025                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1026                         if (heap_socket < 0)
1027                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1028
1029                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1030                                         rte_mbuf_best_mempool_ops());
1031                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1032                                         mb_mempool_cache, 0, mbuf_seg_size,
1033                                         heap_socket);
1034                         break;
1035                 }
1036         case MP_ALLOC_XBUF:
1037                 {
1038                         struct rte_pktmbuf_extmem *ext_mem;
1039                         unsigned int ext_num;
1040
1041                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1042                                                socket_id, pool_name, &ext_mem);
1043                         if (ext_num == 0)
1044                                 rte_exit(EXIT_FAILURE,
1045                                          "Can't create pinned data buffers\n");
1046
1047                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1048                                         rte_mbuf_best_mempool_ops());
1049                         rte_mp = rte_pktmbuf_pool_create_extbuf
1050                                         (pool_name, nb_mbuf, mb_mempool_cache,
1051                                          0, mbuf_seg_size, socket_id,
1052                                          ext_mem, ext_num);
1053                         free(ext_mem);
1054                         break;
1055                 }
1056         default:
1057                 {
1058                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1059                 }
1060         }
1061
1062 err:
1063         if (rte_mp == NULL) {
1064                 rte_exit(EXIT_FAILURE,
1065                         "Creation of mbuf pool for socket %u failed: %s\n",
1066                         socket_id, rte_strerror(rte_errno));
1067         } else if (verbose_level > 0) {
1068                 rte_mempool_dump(stdout, rte_mp);
1069         }
1070         return rte_mp;
1071 }
1072
1073 /*
1074  * Check given socket id is valid or not with NUMA mode,
1075  * if valid, return 0, else return -1
1076  */
1077 static int
1078 check_socket_id(const unsigned int socket_id)
1079 {
1080         static int warning_once = 0;
1081
1082         if (new_socket_id(socket_id)) {
1083                 if (!warning_once && numa_support)
1084                         printf("Warning: NUMA should be configured manually by"
1085                                " using --port-numa-config and"
1086                                " --ring-numa-config parameters along with"
1087                                " --numa.\n");
1088                 warning_once = 1;
1089                 return -1;
1090         }
1091         return 0;
1092 }
1093
1094 /*
1095  * Get the allowed maximum number of RX queues.
1096  * *pid return the port id which has minimal value of
1097  * max_rx_queues in all ports.
1098  */
1099 queueid_t
1100 get_allowed_max_nb_rxq(portid_t *pid)
1101 {
1102         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1103         bool max_rxq_valid = false;
1104         portid_t pi;
1105         struct rte_eth_dev_info dev_info;
1106
1107         RTE_ETH_FOREACH_DEV(pi) {
1108                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1109                         continue;
1110
1111                 max_rxq_valid = true;
1112                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1113                         allowed_max_rxq = dev_info.max_rx_queues;
1114                         *pid = pi;
1115                 }
1116         }
1117         return max_rxq_valid ? allowed_max_rxq : 0;
1118 }
1119
1120 /*
1121  * Check input rxq is valid or not.
1122  * If input rxq is not greater than any of maximum number
1123  * of RX queues of all ports, it is valid.
1124  * if valid, return 0, else return -1
1125  */
1126 int
1127 check_nb_rxq(queueid_t rxq)
1128 {
1129         queueid_t allowed_max_rxq;
1130         portid_t pid = 0;
1131
1132         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1133         if (rxq > allowed_max_rxq) {
1134                 printf("Fail: input rxq (%u) can't be greater "
1135                        "than max_rx_queues (%u) of port %u\n",
1136                        rxq,
1137                        allowed_max_rxq,
1138                        pid);
1139                 return -1;
1140         }
1141         return 0;
1142 }
1143
1144 /*
1145  * Get the allowed maximum number of TX queues.
1146  * *pid return the port id which has minimal value of
1147  * max_tx_queues in all ports.
1148  */
1149 queueid_t
1150 get_allowed_max_nb_txq(portid_t *pid)
1151 {
1152         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1153         bool max_txq_valid = false;
1154         portid_t pi;
1155         struct rte_eth_dev_info dev_info;
1156
1157         RTE_ETH_FOREACH_DEV(pi) {
1158                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1159                         continue;
1160
1161                 max_txq_valid = true;
1162                 if (dev_info.max_tx_queues < allowed_max_txq) {
1163                         allowed_max_txq = dev_info.max_tx_queues;
1164                         *pid = pi;
1165                 }
1166         }
1167         return max_txq_valid ? allowed_max_txq : 0;
1168 }
1169
1170 /*
1171  * Check input txq is valid or not.
1172  * If input txq is not greater than any of maximum number
1173  * of TX queues of all ports, it is valid.
1174  * if valid, return 0, else return -1
1175  */
1176 int
1177 check_nb_txq(queueid_t txq)
1178 {
1179         queueid_t allowed_max_txq;
1180         portid_t pid = 0;
1181
1182         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1183         if (txq > allowed_max_txq) {
1184                 printf("Fail: input txq (%u) can't be greater "
1185                        "than max_tx_queues (%u) of port %u\n",
1186                        txq,
1187                        allowed_max_txq,
1188                        pid);
1189                 return -1;
1190         }
1191         return 0;
1192 }
1193
1194 /*
1195  * Get the allowed maximum number of RXDs of every rx queue.
1196  * *pid return the port id which has minimal value of
1197  * max_rxd in all queues of all ports.
1198  */
1199 static uint16_t
1200 get_allowed_max_nb_rxd(portid_t *pid)
1201 {
1202         uint16_t allowed_max_rxd = UINT16_MAX;
1203         portid_t pi;
1204         struct rte_eth_dev_info dev_info;
1205
1206         RTE_ETH_FOREACH_DEV(pi) {
1207                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1208                         continue;
1209
1210                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1211                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1212                         *pid = pi;
1213                 }
1214         }
1215         return allowed_max_rxd;
1216 }
1217
1218 /*
1219  * Get the allowed minimal number of RXDs of every rx queue.
1220  * *pid return the port id which has minimal value of
1221  * min_rxd in all queues of all ports.
1222  */
1223 static uint16_t
1224 get_allowed_min_nb_rxd(portid_t *pid)
1225 {
1226         uint16_t allowed_min_rxd = 0;
1227         portid_t pi;
1228         struct rte_eth_dev_info dev_info;
1229
1230         RTE_ETH_FOREACH_DEV(pi) {
1231                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1232                         continue;
1233
1234                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1235                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1236                         *pid = pi;
1237                 }
1238         }
1239
1240         return allowed_min_rxd;
1241 }
1242
1243 /*
1244  * Check input rxd is valid or not.
1245  * If input rxd is not greater than any of maximum number
1246  * of RXDs of every Rx queues and is not less than any of
1247  * minimal number of RXDs of every Rx queues, it is valid.
1248  * if valid, return 0, else return -1
1249  */
1250 int
1251 check_nb_rxd(queueid_t rxd)
1252 {
1253         uint16_t allowed_max_rxd;
1254         uint16_t allowed_min_rxd;
1255         portid_t pid = 0;
1256
1257         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1258         if (rxd > allowed_max_rxd) {
1259                 printf("Fail: input rxd (%u) can't be greater "
1260                        "than max_rxds (%u) of port %u\n",
1261                        rxd,
1262                        allowed_max_rxd,
1263                        pid);
1264                 return -1;
1265         }
1266
1267         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1268         if (rxd < allowed_min_rxd) {
1269                 printf("Fail: input rxd (%u) can't be less "
1270                        "than min_rxds (%u) of port %u\n",
1271                        rxd,
1272                        allowed_min_rxd,
1273                        pid);
1274                 return -1;
1275         }
1276
1277         return 0;
1278 }
1279
1280 /*
1281  * Get the allowed maximum number of TXDs of every rx queues.
1282  * *pid return the port id which has minimal value of
1283  * max_txd in every tx queue.
1284  */
1285 static uint16_t
1286 get_allowed_max_nb_txd(portid_t *pid)
1287 {
1288         uint16_t allowed_max_txd = UINT16_MAX;
1289         portid_t pi;
1290         struct rte_eth_dev_info dev_info;
1291
1292         RTE_ETH_FOREACH_DEV(pi) {
1293                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1294                         continue;
1295
1296                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1297                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1298                         *pid = pi;
1299                 }
1300         }
1301         return allowed_max_txd;
1302 }
1303
1304 /*
1305  * Get the allowed maximum number of TXDs of every tx queues.
1306  * *pid return the port id which has minimal value of
1307  * min_txd in every tx queue.
1308  */
1309 static uint16_t
1310 get_allowed_min_nb_txd(portid_t *pid)
1311 {
1312         uint16_t allowed_min_txd = 0;
1313         portid_t pi;
1314         struct rte_eth_dev_info dev_info;
1315
1316         RTE_ETH_FOREACH_DEV(pi) {
1317                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1318                         continue;
1319
1320                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1321                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1322                         *pid = pi;
1323                 }
1324         }
1325
1326         return allowed_min_txd;
1327 }
1328
1329 /*
1330  * Check input txd is valid or not.
1331  * If input txd is not greater than any of maximum number
1332  * of TXDs of every Rx queues, it is valid.
1333  * if valid, return 0, else return -1
1334  */
1335 int
1336 check_nb_txd(queueid_t txd)
1337 {
1338         uint16_t allowed_max_txd;
1339         uint16_t allowed_min_txd;
1340         portid_t pid = 0;
1341
1342         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1343         if (txd > allowed_max_txd) {
1344                 printf("Fail: input txd (%u) can't be greater "
1345                        "than max_txds (%u) of port %u\n",
1346                        txd,
1347                        allowed_max_txd,
1348                        pid);
1349                 return -1;
1350         }
1351
1352         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1353         if (txd < allowed_min_txd) {
1354                 printf("Fail: input txd (%u) can't be less "
1355                        "than min_txds (%u) of port %u\n",
1356                        txd,
1357                        allowed_min_txd,
1358                        pid);
1359                 return -1;
1360         }
1361         return 0;
1362 }
1363
1364
1365 /*
1366  * Get the allowed maximum number of hairpin queues.
1367  * *pid return the port id which has minimal value of
1368  * max_hairpin_queues in all ports.
1369  */
1370 queueid_t
1371 get_allowed_max_nb_hairpinq(portid_t *pid)
1372 {
1373         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1374         portid_t pi;
1375         struct rte_eth_hairpin_cap cap;
1376
1377         RTE_ETH_FOREACH_DEV(pi) {
1378                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1379                         *pid = pi;
1380                         return 0;
1381                 }
1382                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1383                         allowed_max_hairpinq = cap.max_nb_queues;
1384                         *pid = pi;
1385                 }
1386         }
1387         return allowed_max_hairpinq;
1388 }
1389
1390 /*
1391  * Check input hairpin is valid or not.
1392  * If input hairpin is not greater than any of maximum number
1393  * of hairpin queues of all ports, it is valid.
1394  * if valid, return 0, else return -1
1395  */
1396 int
1397 check_nb_hairpinq(queueid_t hairpinq)
1398 {
1399         queueid_t allowed_max_hairpinq;
1400         portid_t pid = 0;
1401
1402         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1403         if (hairpinq > allowed_max_hairpinq) {
1404                 printf("Fail: input hairpin (%u) can't be greater "
1405                        "than max_hairpin_queues (%u) of port %u\n",
1406                        hairpinq, allowed_max_hairpinq, pid);
1407                 return -1;
1408         }
1409         return 0;
1410 }
1411
1412 static void
1413 init_config(void)
1414 {
1415         portid_t pid;
1416         struct rte_port *port;
1417         struct rte_mempool *mbp;
1418         unsigned int nb_mbuf_per_pool;
1419         lcoreid_t  lc_id;
1420         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1421         struct rte_gro_param gro_param;
1422         uint32_t gso_types;
1423         uint16_t data_size;
1424         bool warning = 0;
1425         int k;
1426         int ret;
1427
1428         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1429
1430         /* Configuration of logical cores. */
1431         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1432                                 sizeof(struct fwd_lcore *) * nb_lcores,
1433                                 RTE_CACHE_LINE_SIZE);
1434         if (fwd_lcores == NULL) {
1435                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1436                                                         "failed\n", nb_lcores);
1437         }
1438         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1439                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1440                                                sizeof(struct fwd_lcore),
1441                                                RTE_CACHE_LINE_SIZE);
1442                 if (fwd_lcores[lc_id] == NULL) {
1443                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1444                                                                 "failed\n");
1445                 }
1446                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1447         }
1448
1449         RTE_ETH_FOREACH_DEV(pid) {
1450                 port = &ports[pid];
1451                 /* Apply default TxRx configuration for all ports */
1452                 port->dev_conf.txmode = tx_mode;
1453                 port->dev_conf.rxmode = rx_mode;
1454
1455                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1456                 if (ret != 0)
1457                         rte_exit(EXIT_FAILURE,
1458                                  "rte_eth_dev_info_get() failed\n");
1459
1460                 if (!(port->dev_info.tx_offload_capa &
1461                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1462                         port->dev_conf.txmode.offloads &=
1463                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1464                 if (numa_support) {
1465                         if (port_numa[pid] != NUMA_NO_CONFIG)
1466                                 port_per_socket[port_numa[pid]]++;
1467                         else {
1468                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1469
1470                                 /*
1471                                  * if socket_id is invalid,
1472                                  * set to the first available socket.
1473                                  */
1474                                 if (check_socket_id(socket_id) < 0)
1475                                         socket_id = socket_ids[0];
1476                                 port_per_socket[socket_id]++;
1477                         }
1478                 }
1479
1480                 /* Apply Rx offloads configuration */
1481                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1482                         port->rx_conf[k].offloads =
1483                                 port->dev_conf.rxmode.offloads;
1484                 /* Apply Tx offloads configuration */
1485                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1486                         port->tx_conf[k].offloads =
1487                                 port->dev_conf.txmode.offloads;
1488
1489                 /* set flag to initialize port/queue */
1490                 port->need_reconfig = 1;
1491                 port->need_reconfig_queues = 1;
1492                 port->tx_metadata = 0;
1493
1494                 /* Check for maximum number of segments per MTU. Accordingly
1495                  * update the mbuf data size.
1496                  */
1497                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1498                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1499                         data_size = rx_mode.max_rx_pkt_len /
1500                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1501
1502                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1503                                                         mbuf_data_size[0]) {
1504                                 mbuf_data_size[0] = data_size +
1505                                                  RTE_PKTMBUF_HEADROOM;
1506                                 warning = 1;
1507                         }
1508                 }
1509         }
1510
1511         if (warning)
1512                 TESTPMD_LOG(WARNING,
1513                             "Configured mbuf size of the first segment %hu\n",
1514                             mbuf_data_size[0]);
1515         /*
1516          * Create pools of mbuf.
1517          * If NUMA support is disabled, create a single pool of mbuf in
1518          * socket 0 memory by default.
1519          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1520          *
1521          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1522          * nb_txd can be configured at run time.
1523          */
1524         if (param_total_num_mbufs)
1525                 nb_mbuf_per_pool = param_total_num_mbufs;
1526         else {
1527                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1528                         (nb_lcores * mb_mempool_cache) +
1529                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1530                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1531         }
1532
1533         if (numa_support) {
1534                 uint8_t i, j;
1535
1536                 for (i = 0; i < num_sockets; i++)
1537                         for (j = 0; j < mbuf_data_size_n; j++)
1538                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1539                                         mbuf_pool_create(mbuf_data_size[j],
1540                                                           nb_mbuf_per_pool,
1541                                                           socket_ids[i], j);
1542         } else {
1543                 uint8_t i;
1544
1545                 for (i = 0; i < mbuf_data_size_n; i++)
1546                         mempools[i] = mbuf_pool_create
1547                                         (mbuf_data_size[i],
1548                                          nb_mbuf_per_pool,
1549                                          socket_num == UMA_NO_CONFIG ?
1550                                          0 : socket_num, i);
1551         }
1552
1553         init_port_config();
1554
1555         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1556                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1557         /*
1558          * Records which Mbuf pool to use by each logical core, if needed.
1559          */
1560         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1561                 mbp = mbuf_pool_find(
1562                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1563
1564                 if (mbp == NULL)
1565                         mbp = mbuf_pool_find(0, 0);
1566                 fwd_lcores[lc_id]->mbp = mbp;
1567                 /* initialize GSO context */
1568                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1569                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1570                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1571                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1572                         RTE_ETHER_CRC_LEN;
1573                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1574         }
1575
1576         /* Configuration of packet forwarding streams. */
1577         if (init_fwd_streams() < 0)
1578                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1579
1580         fwd_config_setup();
1581
1582         /* create a gro context for each lcore */
1583         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1584         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1585         gro_param.max_item_per_flow = MAX_PKT_BURST;
1586         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1587                 gro_param.socket_id = rte_lcore_to_socket_id(
1588                                 fwd_lcores_cpuids[lc_id]);
1589                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1590                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1591                         rte_exit(EXIT_FAILURE,
1592                                         "rte_gro_ctx_create() failed\n");
1593                 }
1594         }
1595 }
1596
1597
1598 void
1599 reconfig(portid_t new_port_id, unsigned socket_id)
1600 {
1601         struct rte_port *port;
1602         int ret;
1603
1604         /* Reconfiguration of Ethernet ports. */
1605         port = &ports[new_port_id];
1606
1607         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1608         if (ret != 0)
1609                 return;
1610
1611         /* set flag to initialize port/queue */
1612         port->need_reconfig = 1;
1613         port->need_reconfig_queues = 1;
1614         port->socket_id = socket_id;
1615
1616         init_port_config();
1617 }
1618
1619
1620 int
1621 init_fwd_streams(void)
1622 {
1623         portid_t pid;
1624         struct rte_port *port;
1625         streamid_t sm_id, nb_fwd_streams_new;
1626         queueid_t q;
1627
1628         /* set socket id according to numa or not */
1629         RTE_ETH_FOREACH_DEV(pid) {
1630                 port = &ports[pid];
1631                 if (nb_rxq > port->dev_info.max_rx_queues) {
1632                         printf("Fail: nb_rxq(%d) is greater than "
1633                                 "max_rx_queues(%d)\n", nb_rxq,
1634                                 port->dev_info.max_rx_queues);
1635                         return -1;
1636                 }
1637                 if (nb_txq > port->dev_info.max_tx_queues) {
1638                         printf("Fail: nb_txq(%d) is greater than "
1639                                 "max_tx_queues(%d)\n", nb_txq,
1640                                 port->dev_info.max_tx_queues);
1641                         return -1;
1642                 }
1643                 if (numa_support) {
1644                         if (port_numa[pid] != NUMA_NO_CONFIG)
1645                                 port->socket_id = port_numa[pid];
1646                         else {
1647                                 port->socket_id = rte_eth_dev_socket_id(pid);
1648
1649                                 /*
1650                                  * if socket_id is invalid,
1651                                  * set to the first available socket.
1652                                  */
1653                                 if (check_socket_id(port->socket_id) < 0)
1654                                         port->socket_id = socket_ids[0];
1655                         }
1656                 }
1657                 else {
1658                         if (socket_num == UMA_NO_CONFIG)
1659                                 port->socket_id = 0;
1660                         else
1661                                 port->socket_id = socket_num;
1662                 }
1663         }
1664
1665         q = RTE_MAX(nb_rxq, nb_txq);
1666         if (q == 0) {
1667                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1668                 return -1;
1669         }
1670         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1671         if (nb_fwd_streams_new == nb_fwd_streams)
1672                 return 0;
1673         /* clear the old */
1674         if (fwd_streams != NULL) {
1675                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1676                         if (fwd_streams[sm_id] == NULL)
1677                                 continue;
1678                         rte_free(fwd_streams[sm_id]);
1679                         fwd_streams[sm_id] = NULL;
1680                 }
1681                 rte_free(fwd_streams);
1682                 fwd_streams = NULL;
1683         }
1684
1685         /* init new */
1686         nb_fwd_streams = nb_fwd_streams_new;
1687         if (nb_fwd_streams) {
1688                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1689                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1690                         RTE_CACHE_LINE_SIZE);
1691                 if (fwd_streams == NULL)
1692                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1693                                  " (struct fwd_stream *)) failed\n",
1694                                  nb_fwd_streams);
1695
1696                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1697                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1698                                 " struct fwd_stream", sizeof(struct fwd_stream),
1699                                 RTE_CACHE_LINE_SIZE);
1700                         if (fwd_streams[sm_id] == NULL)
1701                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1702                                          "(struct fwd_stream) failed\n");
1703                 }
1704         }
1705
1706         return 0;
1707 }
1708
1709 static void
1710 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1711 {
1712         uint64_t total_burst, sburst;
1713         uint64_t nb_burst;
1714         uint64_t burst_stats[4];
1715         uint16_t pktnb_stats[4];
1716         uint16_t nb_pkt;
1717         int burst_percent[4], sburstp;
1718         int i;
1719
1720         /*
1721          * First compute the total number of packet bursts and the
1722          * two highest numbers of bursts of the same number of packets.
1723          */
1724         memset(&burst_stats, 0x0, sizeof(burst_stats));
1725         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1726
1727         /* Show stats for 0 burst size always */
1728         total_burst = pbs->pkt_burst_spread[0];
1729         burst_stats[0] = pbs->pkt_burst_spread[0];
1730         pktnb_stats[0] = 0;
1731
1732         /* Find the next 2 burst sizes with highest occurrences. */
1733         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1734                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1735
1736                 if (nb_burst == 0)
1737                         continue;
1738
1739                 total_burst += nb_burst;
1740
1741                 if (nb_burst > burst_stats[1]) {
1742                         burst_stats[2] = burst_stats[1];
1743                         pktnb_stats[2] = pktnb_stats[1];
1744                         burst_stats[1] = nb_burst;
1745                         pktnb_stats[1] = nb_pkt;
1746                 } else if (nb_burst > burst_stats[2]) {
1747                         burst_stats[2] = nb_burst;
1748                         pktnb_stats[2] = nb_pkt;
1749                 }
1750         }
1751         if (total_burst == 0)
1752                 return;
1753
1754         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1755         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1756                 if (i == 3) {
1757                         printf("%d%% of other]\n", 100 - sburstp);
1758                         return;
1759                 }
1760
1761                 sburst += burst_stats[i];
1762                 if (sburst == total_burst) {
1763                         printf("%d%% of %d pkts]\n",
1764                                 100 - sburstp, (int) pktnb_stats[i]);
1765                         return;
1766                 }
1767
1768                 burst_percent[i] =
1769                         (double)burst_stats[i] / total_burst * 100;
1770                 printf("%d%% of %d pkts + ",
1771                         burst_percent[i], (int) pktnb_stats[i]);
1772                 sburstp += burst_percent[i];
1773         }
1774 }
1775
1776 static void
1777 fwd_stream_stats_display(streamid_t stream_id)
1778 {
1779         struct fwd_stream *fs;
1780         static const char *fwd_top_stats_border = "-------";
1781
1782         fs = fwd_streams[stream_id];
1783         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1784             (fs->fwd_dropped == 0))
1785                 return;
1786         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1787                "TX Port=%2d/Queue=%2d %s\n",
1788                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1789                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1790         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1791                " TX-dropped: %-14"PRIu64,
1792                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1793
1794         /* if checksum mode */
1795         if (cur_fwd_eng == &csum_fwd_engine) {
1796                 printf("  RX- bad IP checksum: %-14"PRIu64
1797                        "  Rx- bad L4 checksum: %-14"PRIu64
1798                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1799                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1800                         fs->rx_bad_outer_l4_csum);
1801         } else {
1802                 printf("\n");
1803         }
1804
1805         if (record_burst_stats) {
1806                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1807                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1808         }
1809 }
1810
1811 void
1812 fwd_stats_display(void)
1813 {
1814         static const char *fwd_stats_border = "----------------------";
1815         static const char *acc_stats_border = "+++++++++++++++";
1816         struct {
1817                 struct fwd_stream *rx_stream;
1818                 struct fwd_stream *tx_stream;
1819                 uint64_t tx_dropped;
1820                 uint64_t rx_bad_ip_csum;
1821                 uint64_t rx_bad_l4_csum;
1822                 uint64_t rx_bad_outer_l4_csum;
1823         } ports_stats[RTE_MAX_ETHPORTS];
1824         uint64_t total_rx_dropped = 0;
1825         uint64_t total_tx_dropped = 0;
1826         uint64_t total_rx_nombuf = 0;
1827         struct rte_eth_stats stats;
1828         uint64_t fwd_cycles = 0;
1829         uint64_t total_recv = 0;
1830         uint64_t total_xmit = 0;
1831         struct rte_port *port;
1832         streamid_t sm_id;
1833         portid_t pt_id;
1834         int i;
1835
1836         memset(ports_stats, 0, sizeof(ports_stats));
1837
1838         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1839                 struct fwd_stream *fs = fwd_streams[sm_id];
1840
1841                 if (cur_fwd_config.nb_fwd_streams >
1842                     cur_fwd_config.nb_fwd_ports) {
1843                         fwd_stream_stats_display(sm_id);
1844                 } else {
1845                         ports_stats[fs->tx_port].tx_stream = fs;
1846                         ports_stats[fs->rx_port].rx_stream = fs;
1847                 }
1848
1849                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1850
1851                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1852                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1853                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1854                                 fs->rx_bad_outer_l4_csum;
1855
1856                 if (record_core_cycles)
1857                         fwd_cycles += fs->core_cycles;
1858         }
1859         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1860                 uint8_t j;
1861
1862                 pt_id = fwd_ports_ids[i];
1863                 port = &ports[pt_id];
1864
1865                 rte_eth_stats_get(pt_id, &stats);
1866                 stats.ipackets -= port->stats.ipackets;
1867                 stats.opackets -= port->stats.opackets;
1868                 stats.ibytes -= port->stats.ibytes;
1869                 stats.obytes -= port->stats.obytes;
1870                 stats.imissed -= port->stats.imissed;
1871                 stats.oerrors -= port->stats.oerrors;
1872                 stats.rx_nombuf -= port->stats.rx_nombuf;
1873
1874                 total_recv += stats.ipackets;
1875                 total_xmit += stats.opackets;
1876                 total_rx_dropped += stats.imissed;
1877                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1878                 total_tx_dropped += stats.oerrors;
1879                 total_rx_nombuf  += stats.rx_nombuf;
1880
1881                 printf("\n  %s Forward statistics for port %-2d %s\n",
1882                        fwd_stats_border, pt_id, fwd_stats_border);
1883
1884                 if (!port->rx_queue_stats_mapping_enabled &&
1885                     !port->tx_queue_stats_mapping_enabled) {
1886                         printf("  RX-packets: %-14"PRIu64
1887                                " RX-dropped: %-14"PRIu64
1888                                "RX-total: %-"PRIu64"\n",
1889                                stats.ipackets, stats.imissed,
1890                                stats.ipackets + stats.imissed);
1891
1892                         if (cur_fwd_eng == &csum_fwd_engine)
1893                                 printf("  Bad-ipcsum: %-14"PRIu64
1894                                        " Bad-l4csum: %-14"PRIu64
1895                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1896                                        ports_stats[pt_id].rx_bad_ip_csum,
1897                                        ports_stats[pt_id].rx_bad_l4_csum,
1898                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1899                         if (stats.ierrors + stats.rx_nombuf > 0) {
1900                                 printf("  RX-error: %-"PRIu64"\n",
1901                                        stats.ierrors);
1902                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1903                                        stats.rx_nombuf);
1904                         }
1905
1906                         printf("  TX-packets: %-14"PRIu64
1907                                " TX-dropped: %-14"PRIu64
1908                                "TX-total: %-"PRIu64"\n",
1909                                stats.opackets, ports_stats[pt_id].tx_dropped,
1910                                stats.opackets + ports_stats[pt_id].tx_dropped);
1911                 } else {
1912                         printf("  RX-packets:             %14"PRIu64
1913                                "    RX-dropped:%14"PRIu64
1914                                "    RX-total:%14"PRIu64"\n",
1915                                stats.ipackets, stats.imissed,
1916                                stats.ipackets + stats.imissed);
1917
1918                         if (cur_fwd_eng == &csum_fwd_engine)
1919                                 printf("  Bad-ipcsum:%14"PRIu64
1920                                        "    Bad-l4csum:%14"PRIu64
1921                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1922                                        ports_stats[pt_id].rx_bad_ip_csum,
1923                                        ports_stats[pt_id].rx_bad_l4_csum,
1924                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1925                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1926                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1927                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1928                                        stats.rx_nombuf);
1929                         }
1930
1931                         printf("  TX-packets:             %14"PRIu64
1932                                "    TX-dropped:%14"PRIu64
1933                                "    TX-total:%14"PRIu64"\n",
1934                                stats.opackets, ports_stats[pt_id].tx_dropped,
1935                                stats.opackets + ports_stats[pt_id].tx_dropped);
1936                 }
1937
1938                 if (record_burst_stats) {
1939                         if (ports_stats[pt_id].rx_stream)
1940                                 pkt_burst_stats_display("RX",
1941                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1942                         if (ports_stats[pt_id].tx_stream)
1943                                 pkt_burst_stats_display("TX",
1944                                         &ports_stats[pt_id].tx_stream->tx_burst_stats);
1945                 }
1946
1947                 if (port->rx_queue_stats_mapping_enabled) {
1948                         printf("\n");
1949                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1950                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1951                                        "     RX-errors:%14"PRIu64
1952                                        "    RX-bytes:%14"PRIu64"\n",
1953                                        j, stats.q_ipackets[j],
1954                                        stats.q_errors[j], stats.q_ibytes[j]);
1955                         }
1956                         printf("\n");
1957                 }
1958                 if (port->tx_queue_stats_mapping_enabled) {
1959                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1960                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1961                                        "                                 TX-bytes:%14"
1962                                        PRIu64"\n",
1963                                        j, stats.q_opackets[j],
1964                                        stats.q_obytes[j]);
1965                         }
1966                 }
1967
1968                 printf("  %s--------------------------------%s\n",
1969                        fwd_stats_border, fwd_stats_border);
1970         }
1971
1972         printf("\n  %s Accumulated forward statistics for all ports"
1973                "%s\n",
1974                acc_stats_border, acc_stats_border);
1975         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1976                "%-"PRIu64"\n"
1977                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1978                "%-"PRIu64"\n",
1979                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1980                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1981         if (total_rx_nombuf > 0)
1982                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1983         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1984                "%s\n",
1985                acc_stats_border, acc_stats_border);
1986         if (record_core_cycles) {
1987 #define CYC_PER_MHZ 1E6
1988                 if (total_recv > 0 || total_xmit > 0) {
1989                         uint64_t total_pkts = 0;
1990                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1991                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1992                                 total_pkts = total_xmit;
1993                         else
1994                                 total_pkts = total_recv;
1995
1996                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1997                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1998                                " MHz Clock\n",
1999                                (double) fwd_cycles / total_pkts,
2000                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2001                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2002                 }
2003         }
2004 }
2005
2006 void
2007 fwd_stats_reset(void)
2008 {
2009         streamid_t sm_id;
2010         portid_t pt_id;
2011         int i;
2012
2013         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2014                 pt_id = fwd_ports_ids[i];
2015                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2016         }
2017         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2018                 struct fwd_stream *fs = fwd_streams[sm_id];
2019
2020                 fs->rx_packets = 0;
2021                 fs->tx_packets = 0;
2022                 fs->fwd_dropped = 0;
2023                 fs->rx_bad_ip_csum = 0;
2024                 fs->rx_bad_l4_csum = 0;
2025                 fs->rx_bad_outer_l4_csum = 0;
2026
2027                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2028                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2029                 fs->core_cycles = 0;
2030         }
2031 }
2032
2033 static void
2034 flush_fwd_rx_queues(void)
2035 {
2036         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2037         portid_t  rxp;
2038         portid_t port_id;
2039         queueid_t rxq;
2040         uint16_t  nb_rx;
2041         uint16_t  i;
2042         uint8_t   j;
2043         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2044         uint64_t timer_period;
2045
2046         /* convert to number of cycles */
2047         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2048
2049         for (j = 0; j < 2; j++) {
2050                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2051                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2052                                 port_id = fwd_ports_ids[rxp];
2053                                 /**
2054                                 * testpmd can stuck in the below do while loop
2055                                 * if rte_eth_rx_burst() always returns nonzero
2056                                 * packets. So timer is added to exit this loop
2057                                 * after 1sec timer expiry.
2058                                 */
2059                                 prev_tsc = rte_rdtsc();
2060                                 do {
2061                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2062                                                 pkts_burst, MAX_PKT_BURST);
2063                                         for (i = 0; i < nb_rx; i++)
2064                                                 rte_pktmbuf_free(pkts_burst[i]);
2065
2066                                         cur_tsc = rte_rdtsc();
2067                                         diff_tsc = cur_tsc - prev_tsc;
2068                                         timer_tsc += diff_tsc;
2069                                 } while ((nb_rx > 0) &&
2070                                         (timer_tsc < timer_period));
2071                                 timer_tsc = 0;
2072                         }
2073                 }
2074                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2075         }
2076 }
2077
2078 static void
2079 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2080 {
2081         struct fwd_stream **fsm;
2082         streamid_t nb_fs;
2083         streamid_t sm_id;
2084 #ifdef RTE_LIBRTE_BITRATESTATS
2085         uint64_t tics_per_1sec;
2086         uint64_t tics_datum;
2087         uint64_t tics_current;
2088         uint16_t i, cnt_ports;
2089
2090         cnt_ports = nb_ports;
2091         tics_datum = rte_rdtsc();
2092         tics_per_1sec = rte_get_timer_hz();
2093 #endif
2094         fsm = &fwd_streams[fc->stream_idx];
2095         nb_fs = fc->stream_nb;
2096         do {
2097                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2098                         (*pkt_fwd)(fsm[sm_id]);
2099 #ifdef RTE_LIBRTE_BITRATESTATS
2100                 if (bitrate_enabled != 0 &&
2101                                 bitrate_lcore_id == rte_lcore_id()) {
2102                         tics_current = rte_rdtsc();
2103                         if (tics_current - tics_datum >= tics_per_1sec) {
2104                                 /* Periodic bitrate calculation */
2105                                 for (i = 0; i < cnt_ports; i++)
2106                                         rte_stats_bitrate_calc(bitrate_data,
2107                                                 ports_ids[i]);
2108                                 tics_datum = tics_current;
2109                         }
2110                 }
2111 #endif
2112 #ifdef RTE_LIBRTE_LATENCY_STATS
2113                 if (latencystats_enabled != 0 &&
2114                                 latencystats_lcore_id == rte_lcore_id())
2115                         rte_latencystats_update();
2116 #endif
2117
2118         } while (! fc->stopped);
2119 }
2120
2121 static int
2122 start_pkt_forward_on_core(void *fwd_arg)
2123 {
2124         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2125                              cur_fwd_config.fwd_eng->packet_fwd);
2126         return 0;
2127 }
2128
2129 /*
2130  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2131  * Used to start communication flows in network loopback test configurations.
2132  */
2133 static int
2134 run_one_txonly_burst_on_core(void *fwd_arg)
2135 {
2136         struct fwd_lcore *fwd_lc;
2137         struct fwd_lcore tmp_lcore;
2138
2139         fwd_lc = (struct fwd_lcore *) fwd_arg;
2140         tmp_lcore = *fwd_lc;
2141         tmp_lcore.stopped = 1;
2142         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2143         return 0;
2144 }
2145
2146 /*
2147  * Launch packet forwarding:
2148  *     - Setup per-port forwarding context.
2149  *     - launch logical cores with their forwarding configuration.
2150  */
2151 static void
2152 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2153 {
2154         port_fwd_begin_t port_fwd_begin;
2155         unsigned int i;
2156         unsigned int lc_id;
2157         int diag;
2158
2159         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2160         if (port_fwd_begin != NULL) {
2161                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2162                         (*port_fwd_begin)(fwd_ports_ids[i]);
2163         }
2164         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2165                 lc_id = fwd_lcores_cpuids[i];
2166                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2167                         fwd_lcores[i]->stopped = 0;
2168                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2169                                                      fwd_lcores[i], lc_id);
2170                         if (diag != 0)
2171                                 printf("launch lcore %u failed - diag=%d\n",
2172                                        lc_id, diag);
2173                 }
2174         }
2175 }
2176
2177 /*
2178  * Launch packet forwarding configuration.
2179  */
2180 void
2181 start_packet_forwarding(int with_tx_first)
2182 {
2183         port_fwd_begin_t port_fwd_begin;
2184         port_fwd_end_t  port_fwd_end;
2185         struct rte_port *port;
2186         unsigned int i;
2187         portid_t   pt_id;
2188
2189         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2190                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2191
2192         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2193                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2194
2195         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2196                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2197                 (!nb_rxq || !nb_txq))
2198                 rte_exit(EXIT_FAILURE,
2199                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2200                         cur_fwd_eng->fwd_mode_name);
2201
2202         if (all_ports_started() == 0) {
2203                 printf("Not all ports were started\n");
2204                 return;
2205         }
2206         if (test_done == 0) {
2207                 printf("Packet forwarding already started\n");
2208                 return;
2209         }
2210
2211
2212         if(dcb_test) {
2213                 for (i = 0; i < nb_fwd_ports; i++) {
2214                         pt_id = fwd_ports_ids[i];
2215                         port = &ports[pt_id];
2216                         if (!port->dcb_flag) {
2217                                 printf("In DCB mode, all forwarding ports must "
2218                                        "be configured in this mode.\n");
2219                                 return;
2220                         }
2221                 }
2222                 if (nb_fwd_lcores == 1) {
2223                         printf("In DCB mode,the nb forwarding cores "
2224                                "should be larger than 1.\n");
2225                         return;
2226                 }
2227         }
2228         test_done = 0;
2229
2230         fwd_config_setup();
2231
2232         if(!no_flush_rx)
2233                 flush_fwd_rx_queues();
2234
2235         pkt_fwd_config_display(&cur_fwd_config);
2236         rxtx_config_display();
2237
2238         fwd_stats_reset();
2239         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2240                 pt_id = fwd_ports_ids[i];
2241                 port = &ports[pt_id];
2242                 map_port_queue_stats_mapping_registers(pt_id, port);
2243         }
2244         if (with_tx_first) {
2245                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2246                 if (port_fwd_begin != NULL) {
2247                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2248                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2249                 }
2250                 while (with_tx_first--) {
2251                         launch_packet_forwarding(
2252                                         run_one_txonly_burst_on_core);
2253                         rte_eal_mp_wait_lcore();
2254                 }
2255                 port_fwd_end = tx_only_engine.port_fwd_end;
2256                 if (port_fwd_end != NULL) {
2257                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2258                                 (*port_fwd_end)(fwd_ports_ids[i]);
2259                 }
2260         }
2261         launch_packet_forwarding(start_pkt_forward_on_core);
2262 }
2263
2264 void
2265 stop_packet_forwarding(void)
2266 {
2267         port_fwd_end_t port_fwd_end;
2268         lcoreid_t lc_id;
2269         portid_t pt_id;
2270         int i;
2271
2272         if (test_done) {
2273                 printf("Packet forwarding not started\n");
2274                 return;
2275         }
2276         printf("Telling cores to stop...");
2277         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2278                 fwd_lcores[lc_id]->stopped = 1;
2279         printf("\nWaiting for lcores to finish...\n");
2280         rte_eal_mp_wait_lcore();
2281         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2282         if (port_fwd_end != NULL) {
2283                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2284                         pt_id = fwd_ports_ids[i];
2285                         (*port_fwd_end)(pt_id);
2286                 }
2287         }
2288
2289         fwd_stats_display();
2290
2291         printf("\nDone.\n");
2292         test_done = 1;
2293 }
2294
2295 void
2296 dev_set_link_up(portid_t pid)
2297 {
2298         if (rte_eth_dev_set_link_up(pid) < 0)
2299                 printf("\nSet link up fail.\n");
2300 }
2301
2302 void
2303 dev_set_link_down(portid_t pid)
2304 {
2305         if (rte_eth_dev_set_link_down(pid) < 0)
2306                 printf("\nSet link down fail.\n");
2307 }
2308
2309 static int
2310 all_ports_started(void)
2311 {
2312         portid_t pi;
2313         struct rte_port *port;
2314
2315         RTE_ETH_FOREACH_DEV(pi) {
2316                 port = &ports[pi];
2317                 /* Check if there is a port which is not started */
2318                 if ((port->port_status != RTE_PORT_STARTED) &&
2319                         (port->slave_flag == 0))
2320                         return 0;
2321         }
2322
2323         /* No port is not started */
2324         return 1;
2325 }
2326
2327 int
2328 port_is_stopped(portid_t port_id)
2329 {
2330         struct rte_port *port = &ports[port_id];
2331
2332         if ((port->port_status != RTE_PORT_STOPPED) &&
2333             (port->slave_flag == 0))
2334                 return 0;
2335         return 1;
2336 }
2337
2338 int
2339 all_ports_stopped(void)
2340 {
2341         portid_t pi;
2342
2343         RTE_ETH_FOREACH_DEV(pi) {
2344                 if (!port_is_stopped(pi))
2345                         return 0;
2346         }
2347
2348         return 1;
2349 }
2350
2351 int
2352 port_is_started(portid_t port_id)
2353 {
2354         if (port_id_is_invalid(port_id, ENABLED_WARN))
2355                 return 0;
2356
2357         if (ports[port_id].port_status != RTE_PORT_STARTED)
2358                 return 0;
2359
2360         return 1;
2361 }
2362
2363 /* Configure the Rx and Tx hairpin queues for the selected port. */
2364 static int
2365 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2366 {
2367         queueid_t qi;
2368         struct rte_eth_hairpin_conf hairpin_conf = {
2369                 .peer_count = 1,
2370         };
2371         int i;
2372         int diag;
2373         struct rte_port *port = &ports[pi];
2374         uint16_t peer_rx_port = pi;
2375         uint16_t peer_tx_port = pi;
2376         uint32_t manual = 1;
2377         uint32_t tx_exp = hairpin_mode & 0x10;
2378
2379         if (!(hairpin_mode & 0xf)) {
2380                 peer_rx_port = pi;
2381                 peer_tx_port = pi;
2382                 manual = 0;
2383         } else if (hairpin_mode & 0x1) {
2384                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2385                                                        RTE_ETH_DEV_NO_OWNER);
2386                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2387                         peer_tx_port = rte_eth_find_next_owned_by(0,
2388                                                 RTE_ETH_DEV_NO_OWNER);
2389                 if (p_pi != RTE_MAX_ETHPORTS) {
2390                         peer_rx_port = p_pi;
2391                 } else {
2392                         uint16_t next_pi;
2393
2394                         /* Last port will be the peer RX port of the first. */
2395                         RTE_ETH_FOREACH_DEV(next_pi)
2396                                 peer_rx_port = next_pi;
2397                 }
2398                 manual = 1;
2399         } else if (hairpin_mode & 0x2) {
2400                 if (cnt_pi & 0x1) {
2401                         peer_rx_port = p_pi;
2402                 } else {
2403                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2404                                                 RTE_ETH_DEV_NO_OWNER);
2405                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2406                                 peer_rx_port = pi;
2407                 }
2408                 peer_tx_port = peer_rx_port;
2409                 manual = 1;
2410         }
2411
2412         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2413                 hairpin_conf.peers[0].port = peer_rx_port;
2414                 hairpin_conf.peers[0].queue = i + nb_rxq;
2415                 hairpin_conf.manual_bind = !!manual;
2416                 hairpin_conf.tx_explicit = !!tx_exp;
2417                 diag = rte_eth_tx_hairpin_queue_setup
2418                         (pi, qi, nb_txd, &hairpin_conf);
2419                 i++;
2420                 if (diag == 0)
2421                         continue;
2422
2423                 /* Fail to setup rx queue, return */
2424                 if (rte_atomic16_cmpset(&(port->port_status),
2425                                         RTE_PORT_HANDLING,
2426                                         RTE_PORT_STOPPED) == 0)
2427                         printf("Port %d can not be set back "
2428                                         "to stopped\n", pi);
2429                 printf("Fail to configure port %d hairpin "
2430                                 "queues\n", pi);
2431                 /* try to reconfigure queues next time */
2432                 port->need_reconfig_queues = 1;
2433                 return -1;
2434         }
2435         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2436                 hairpin_conf.peers[0].port = peer_tx_port;
2437                 hairpin_conf.peers[0].queue = i + nb_txq;
2438                 hairpin_conf.manual_bind = !!manual;
2439                 hairpin_conf.tx_explicit = !!tx_exp;
2440                 diag = rte_eth_rx_hairpin_queue_setup
2441                         (pi, qi, nb_rxd, &hairpin_conf);
2442                 i++;
2443                 if (diag == 0)
2444                         continue;
2445
2446                 /* Fail to setup rx queue, return */
2447                 if (rte_atomic16_cmpset(&(port->port_status),
2448                                         RTE_PORT_HANDLING,
2449                                         RTE_PORT_STOPPED) == 0)
2450                         printf("Port %d can not be set back "
2451                                         "to stopped\n", pi);
2452                 printf("Fail to configure port %d hairpin "
2453                                 "queues\n", pi);
2454                 /* try to reconfigure queues next time */
2455                 port->need_reconfig_queues = 1;
2456                 return -1;
2457         }
2458         return 0;
2459 }
2460
2461 /* Configure the Rx with optional split. */
2462 int
2463 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2464                uint16_t nb_rx_desc, unsigned int socket_id,
2465                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2466 {
2467         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2468         unsigned int i, mp_n;
2469         int ret;
2470
2471         if (rx_pkt_nb_segs <= 1 ||
2472             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2473                 rx_conf->rx_seg = NULL;
2474                 rx_conf->rx_nseg = 0;
2475                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2476                                              nb_rx_desc, socket_id,
2477                                              rx_conf, mp);
2478                 return ret;
2479         }
2480         for (i = 0; i < rx_pkt_nb_segs; i++) {
2481                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2482                 struct rte_mempool *mpx;
2483                 /*
2484                  * Use last valid pool for the segments with number
2485                  * exceeding the pool index.
2486                  */
2487                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2488                 mpx = mbuf_pool_find(socket_id, mp_n);
2489                 /* Handle zero as mbuf data buffer size. */
2490                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2491                                    rx_pkt_seg_lengths[i] :
2492                                    mbuf_data_size[mp_n];
2493                 rx_seg->offset = i < rx_pkt_nb_offs ?
2494                                    rx_pkt_seg_offsets[i] : 0;
2495                 rx_seg->mp = mpx ? mpx : mp;
2496         }
2497         rx_conf->rx_nseg = rx_pkt_nb_segs;
2498         rx_conf->rx_seg = rx_useg;
2499         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2500                                     socket_id, rx_conf, NULL);
2501         rx_conf->rx_seg = NULL;
2502         rx_conf->rx_nseg = 0;
2503         return ret;
2504 }
2505
2506 int
2507 start_port(portid_t pid)
2508 {
2509         int diag, need_check_link_status = -1;
2510         portid_t pi;
2511         portid_t p_pi = RTE_MAX_ETHPORTS;
2512         portid_t pl[RTE_MAX_ETHPORTS];
2513         portid_t peer_pl[RTE_MAX_ETHPORTS];
2514         uint16_t cnt_pi = 0;
2515         uint16_t cfg_pi = 0;
2516         int peer_pi;
2517         queueid_t qi;
2518         struct rte_port *port;
2519         struct rte_ether_addr mac_addr;
2520         struct rte_eth_hairpin_cap cap;
2521
2522         if (port_id_is_invalid(pid, ENABLED_WARN))
2523                 return 0;
2524
2525         if(dcb_config)
2526                 dcb_test = 1;
2527         RTE_ETH_FOREACH_DEV(pi) {
2528                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2529                         continue;
2530
2531                 need_check_link_status = 0;
2532                 port = &ports[pi];
2533                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2534                                                  RTE_PORT_HANDLING) == 0) {
2535                         printf("Port %d is now not stopped\n", pi);
2536                         continue;
2537                 }
2538
2539                 if (port->need_reconfig > 0) {
2540                         port->need_reconfig = 0;
2541
2542                         if (flow_isolate_all) {
2543                                 int ret = port_flow_isolate(pi, 1);
2544                                 if (ret) {
2545                                         printf("Failed to apply isolated"
2546                                                " mode on port %d\n", pi);
2547                                         return -1;
2548                                 }
2549                         }
2550                         configure_rxtx_dump_callbacks(0);
2551                         printf("Configuring Port %d (socket %u)\n", pi,
2552                                         port->socket_id);
2553                         if (nb_hairpinq > 0 &&
2554                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2555                                 printf("Port %d doesn't support hairpin "
2556                                        "queues\n", pi);
2557                                 return -1;
2558                         }
2559                         /* configure port */
2560                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2561                                                      nb_txq + nb_hairpinq,
2562                                                      &(port->dev_conf));
2563                         if (diag != 0) {
2564                                 if (rte_atomic16_cmpset(&(port->port_status),
2565                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2566                                         printf("Port %d can not be set back "
2567                                                         "to stopped\n", pi);
2568                                 printf("Fail to configure port %d\n", pi);
2569                                 /* try to reconfigure port next time */
2570                                 port->need_reconfig = 1;
2571                                 return -1;
2572                         }
2573                 }
2574                 if (port->need_reconfig_queues > 0) {
2575                         port->need_reconfig_queues = 0;
2576                         /* setup tx queues */
2577                         for (qi = 0; qi < nb_txq; qi++) {
2578                                 if ((numa_support) &&
2579                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2580                                         diag = rte_eth_tx_queue_setup(pi, qi,
2581                                                 port->nb_tx_desc[qi],
2582                                                 txring_numa[pi],
2583                                                 &(port->tx_conf[qi]));
2584                                 else
2585                                         diag = rte_eth_tx_queue_setup(pi, qi,
2586                                                 port->nb_tx_desc[qi],
2587                                                 port->socket_id,
2588                                                 &(port->tx_conf[qi]));
2589
2590                                 if (diag == 0)
2591                                         continue;
2592
2593                                 /* Fail to setup tx queue, return */
2594                                 if (rte_atomic16_cmpset(&(port->port_status),
2595                                                         RTE_PORT_HANDLING,
2596                                                         RTE_PORT_STOPPED) == 0)
2597                                         printf("Port %d can not be set back "
2598                                                         "to stopped\n", pi);
2599                                 printf("Fail to configure port %d tx queues\n",
2600                                        pi);
2601                                 /* try to reconfigure queues next time */
2602                                 port->need_reconfig_queues = 1;
2603                                 return -1;
2604                         }
2605                         for (qi = 0; qi < nb_rxq; qi++) {
2606                                 /* setup rx queues */
2607                                 if ((numa_support) &&
2608                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2609                                         struct rte_mempool * mp =
2610                                                 mbuf_pool_find
2611                                                         (rxring_numa[pi], 0);
2612                                         if (mp == NULL) {
2613                                                 printf("Failed to setup RX queue:"
2614                                                         "No mempool allocation"
2615                                                         " on the socket %d\n",
2616                                                         rxring_numa[pi]);
2617                                                 return -1;
2618                                         }
2619
2620                                         diag = rx_queue_setup(pi, qi,
2621                                              port->nb_rx_desc[qi],
2622                                              rxring_numa[pi],
2623                                              &(port->rx_conf[qi]),
2624                                              mp);
2625                                 } else {
2626                                         struct rte_mempool *mp =
2627                                                 mbuf_pool_find
2628                                                         (port->socket_id, 0);
2629                                         if (mp == NULL) {
2630                                                 printf("Failed to setup RX queue:"
2631                                                         "No mempool allocation"
2632                                                         " on the socket %d\n",
2633                                                         port->socket_id);
2634                                                 return -1;
2635                                         }
2636                                         diag = rx_queue_setup(pi, qi,
2637                                              port->nb_rx_desc[qi],
2638                                              port->socket_id,
2639                                              &(port->rx_conf[qi]),
2640                                              mp);
2641                                 }
2642                                 if (diag == 0)
2643                                         continue;
2644
2645                                 /* Fail to setup rx queue, return */
2646                                 if (rte_atomic16_cmpset(&(port->port_status),
2647                                                         RTE_PORT_HANDLING,
2648                                                         RTE_PORT_STOPPED) == 0)
2649                                         printf("Port %d can not be set back "
2650                                                         "to stopped\n", pi);
2651                                 printf("Fail to configure port %d rx queues\n",
2652                                        pi);
2653                                 /* try to reconfigure queues next time */
2654                                 port->need_reconfig_queues = 1;
2655                                 return -1;
2656                         }
2657                         /* setup hairpin queues */
2658                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2659                                 return -1;
2660                 }
2661                 configure_rxtx_dump_callbacks(verbose_level);
2662                 if (clear_ptypes) {
2663                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2664                                         NULL, 0);
2665                         if (diag < 0)
2666                                 printf(
2667                                 "Port %d: Failed to disable Ptype parsing\n",
2668                                 pi);
2669                 }
2670
2671                 p_pi = pi;
2672                 cnt_pi++;
2673
2674                 /* start port */
2675                 if (rte_eth_dev_start(pi) < 0) {
2676                         printf("Fail to start port %d\n", pi);
2677
2678                         /* Fail to setup rx queue, return */
2679                         if (rte_atomic16_cmpset(&(port->port_status),
2680                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2681                                 printf("Port %d can not be set back to "
2682                                                         "stopped\n", pi);
2683                         continue;
2684                 }
2685
2686                 if (rte_atomic16_cmpset(&(port->port_status),
2687                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2688                         printf("Port %d can not be set into started\n", pi);
2689
2690                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2691                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2692                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2693                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2694                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2695
2696                 /* at least one port started, need checking link status */
2697                 need_check_link_status = 1;
2698
2699                 pl[cfg_pi++] = pi;
2700         }
2701
2702         if (need_check_link_status == 1 && !no_link_check)
2703                 check_all_ports_link_status(RTE_PORT_ALL);
2704         else if (need_check_link_status == 0)
2705                 printf("Please stop the ports first\n");
2706
2707         if (hairpin_mode & 0xf) {
2708                 uint16_t i;
2709                 int j;
2710
2711                 /* bind all started hairpin ports */
2712                 for (i = 0; i < cfg_pi; i++) {
2713                         pi = pl[i];
2714                         /* bind current Tx to all peer Rx */
2715                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2716                                                         RTE_MAX_ETHPORTS, 1);
2717                         if (peer_pi < 0)
2718                                 return peer_pi;
2719                         for (j = 0; j < peer_pi; j++) {
2720                                 if (!port_is_started(peer_pl[j]))
2721                                         continue;
2722                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2723                                 if (diag < 0) {
2724                                         printf("Error during binding hairpin"
2725                                                " Tx port %u to %u: %s\n",
2726                                                pi, peer_pl[j],
2727                                                rte_strerror(-diag));
2728                                         return -1;
2729                                 }
2730                         }
2731                         /* bind all peer Tx to current Rx */
2732                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2733                                                         RTE_MAX_ETHPORTS, 0);
2734                         if (peer_pi < 0)
2735                                 return peer_pi;
2736                         for (j = 0; j < peer_pi; j++) {
2737                                 if (!port_is_started(peer_pl[j]))
2738                                         continue;
2739                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2740                                 if (diag < 0) {
2741                                         printf("Error during binding hairpin"
2742                                                " Tx port %u to %u: %s\n",
2743                                                peer_pl[j], pi,
2744                                                rte_strerror(-diag));
2745                                         return -1;
2746                                 }
2747                         }
2748                 }
2749         }
2750
2751         printf("Done\n");
2752         return 0;
2753 }
2754
2755 void
2756 stop_port(portid_t pid)
2757 {
2758         portid_t pi;
2759         struct rte_port *port;
2760         int need_check_link_status = 0;
2761         portid_t peer_pl[RTE_MAX_ETHPORTS];
2762         int peer_pi;
2763
2764         if (dcb_test) {
2765                 dcb_test = 0;
2766                 dcb_config = 0;
2767         }
2768
2769         if (port_id_is_invalid(pid, ENABLED_WARN))
2770                 return;
2771
2772         printf("Stopping ports...\n");
2773
2774         RTE_ETH_FOREACH_DEV(pi) {
2775                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2776                         continue;
2777
2778                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2779                         printf("Please remove port %d from forwarding configuration.\n", pi);
2780                         continue;
2781                 }
2782
2783                 if (port_is_bonding_slave(pi)) {
2784                         printf("Please remove port %d from bonded device.\n", pi);
2785                         continue;
2786                 }
2787
2788                 port = &ports[pi];
2789                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2790                                                 RTE_PORT_HANDLING) == 0)
2791                         continue;
2792
2793                 if (hairpin_mode & 0xf) {
2794                         int j;
2795
2796                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2797                         /* unbind all peer Tx from current Rx */
2798                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2799                                                         RTE_MAX_ETHPORTS, 0);
2800                         if (peer_pi < 0)
2801                                 continue;
2802                         for (j = 0; j < peer_pi; j++) {
2803                                 if (!port_is_started(peer_pl[j]))
2804                                         continue;
2805                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2806                         }
2807                 }
2808
2809                 rte_eth_dev_stop(pi);
2810
2811                 if (rte_atomic16_cmpset(&(port->port_status),
2812                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2813                         printf("Port %d can not be set into stopped\n", pi);
2814                 need_check_link_status = 1;
2815         }
2816         if (need_check_link_status && !no_link_check)
2817                 check_all_ports_link_status(RTE_PORT_ALL);
2818
2819         printf("Done\n");
2820 }
2821
2822 static void
2823 remove_invalid_ports_in(portid_t *array, portid_t *total)
2824 {
2825         portid_t i;
2826         portid_t new_total = 0;
2827
2828         for (i = 0; i < *total; i++)
2829                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2830                         array[new_total] = array[i];
2831                         new_total++;
2832                 }
2833         *total = new_total;
2834 }
2835
2836 static void
2837 remove_invalid_ports(void)
2838 {
2839         remove_invalid_ports_in(ports_ids, &nb_ports);
2840         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2841         nb_cfg_ports = nb_fwd_ports;
2842 }
2843
2844 void
2845 close_port(portid_t pid)
2846 {
2847         portid_t pi;
2848         struct rte_port *port;
2849
2850         if (port_id_is_invalid(pid, ENABLED_WARN))
2851                 return;
2852
2853         printf("Closing ports...\n");
2854
2855         RTE_ETH_FOREACH_DEV(pi) {
2856                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2857                         continue;
2858
2859                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2860                         printf("Please remove port %d from forwarding configuration.\n", pi);
2861                         continue;
2862                 }
2863
2864                 if (port_is_bonding_slave(pi)) {
2865                         printf("Please remove port %d from bonded device.\n", pi);
2866                         continue;
2867                 }
2868
2869                 port = &ports[pi];
2870                 if (rte_atomic16_cmpset(&(port->port_status),
2871                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2872                         printf("Port %d is already closed\n", pi);
2873                         continue;
2874                 }
2875
2876                 port_flow_flush(pi);
2877                 rte_eth_dev_close(pi);
2878         }
2879
2880         remove_invalid_ports();
2881         printf("Done\n");
2882 }
2883
2884 void
2885 reset_port(portid_t pid)
2886 {
2887         int diag;
2888         portid_t pi;
2889         struct rte_port *port;
2890
2891         if (port_id_is_invalid(pid, ENABLED_WARN))
2892                 return;
2893
2894         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2895                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2896                 printf("Can not reset port(s), please stop port(s) first.\n");
2897                 return;
2898         }
2899
2900         printf("Resetting ports...\n");
2901
2902         RTE_ETH_FOREACH_DEV(pi) {
2903                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2904                         continue;
2905
2906                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2907                         printf("Please remove port %d from forwarding "
2908                                "configuration.\n", pi);
2909                         continue;
2910                 }
2911
2912                 if (port_is_bonding_slave(pi)) {
2913                         printf("Please remove port %d from bonded device.\n",
2914                                pi);
2915                         continue;
2916                 }
2917
2918                 diag = rte_eth_dev_reset(pi);
2919                 if (diag == 0) {
2920                         port = &ports[pi];
2921                         port->need_reconfig = 1;
2922                         port->need_reconfig_queues = 1;
2923                 } else {
2924                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2925                 }
2926         }
2927
2928         printf("Done\n");
2929 }
2930
2931 void
2932 attach_port(char *identifier)
2933 {
2934         portid_t pi;
2935         struct rte_dev_iterator iterator;
2936
2937         printf("Attaching a new port...\n");
2938
2939         if (identifier == NULL) {
2940                 printf("Invalid parameters are specified\n");
2941                 return;
2942         }
2943
2944         if (rte_dev_probe(identifier) < 0) {
2945                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2946                 return;
2947         }
2948
2949         /* first attach mode: event */
2950         if (setup_on_probe_event) {
2951                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2952                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2953                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2954                                         ports[pi].need_setup != 0)
2955                                 setup_attached_port(pi);
2956                 return;
2957         }
2958
2959         /* second attach mode: iterator */
2960         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2961                 /* setup ports matching the devargs used for probing */
2962                 if (port_is_forwarding(pi))
2963                         continue; /* port was already attached before */
2964                 setup_attached_port(pi);
2965         }
2966 }
2967
2968 static void
2969 setup_attached_port(portid_t pi)
2970 {
2971         unsigned int socket_id;
2972         int ret;
2973
2974         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2975         /* if socket_id is invalid, set to the first available socket. */
2976         if (check_socket_id(socket_id) < 0)
2977                 socket_id = socket_ids[0];
2978         reconfig(pi, socket_id);
2979         ret = rte_eth_promiscuous_enable(pi);
2980         if (ret != 0)
2981                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2982                         pi, rte_strerror(-ret));
2983
2984         ports_ids[nb_ports++] = pi;
2985         fwd_ports_ids[nb_fwd_ports++] = pi;
2986         nb_cfg_ports = nb_fwd_ports;
2987         ports[pi].need_setup = 0;
2988         ports[pi].port_status = RTE_PORT_STOPPED;
2989
2990         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2991         printf("Done\n");
2992 }
2993
2994 static void
2995 detach_device(struct rte_device *dev)
2996 {
2997         portid_t sibling;
2998
2999         if (dev == NULL) {
3000                 printf("Device already removed\n");
3001                 return;
3002         }
3003
3004         printf("Removing a device...\n");
3005
3006         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3007                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3008                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3009                                 printf("Port %u not stopped\n", sibling);
3010                                 return;
3011                         }
3012                         port_flow_flush(sibling);
3013                 }
3014         }
3015
3016         if (rte_dev_remove(dev) < 0) {
3017                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3018                 return;
3019         }
3020         remove_invalid_ports();
3021
3022         printf("Device is detached\n");
3023         printf("Now total ports is %d\n", nb_ports);
3024         printf("Done\n");
3025         return;
3026 }
3027
3028 void
3029 detach_port_device(portid_t port_id)
3030 {
3031         if (port_id_is_invalid(port_id, ENABLED_WARN))
3032                 return;
3033
3034         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3035                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3036                         printf("Port not stopped\n");
3037                         return;
3038                 }
3039                 printf("Port was not closed\n");
3040         }
3041
3042         detach_device(rte_eth_devices[port_id].device);
3043 }
3044
3045 void
3046 detach_devargs(char *identifier)
3047 {
3048         struct rte_dev_iterator iterator;
3049         struct rte_devargs da;
3050         portid_t port_id;
3051
3052         printf("Removing a device...\n");
3053
3054         memset(&da, 0, sizeof(da));
3055         if (rte_devargs_parsef(&da, "%s", identifier)) {
3056                 printf("cannot parse identifier\n");
3057                 if (da.args)
3058                         free(da.args);
3059                 return;
3060         }
3061
3062         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3063                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3064                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3065                                 printf("Port %u not stopped\n", port_id);
3066                                 rte_eth_iterator_cleanup(&iterator);
3067                                 return;
3068                         }
3069                         port_flow_flush(port_id);
3070                 }
3071         }
3072
3073         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3074                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3075                             da.name, da.bus->name);
3076                 return;
3077         }
3078
3079         remove_invalid_ports();
3080
3081         printf("Device %s is detached\n", identifier);
3082         printf("Now total ports is %d\n", nb_ports);
3083         printf("Done\n");
3084 }
3085
3086 void
3087 pmd_test_exit(void)
3088 {
3089         portid_t pt_id;
3090         unsigned int i;
3091         int ret;
3092
3093         if (test_done == 0)
3094                 stop_packet_forwarding();
3095
3096         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3097                 if (mempools[i]) {
3098                         if (mp_alloc_type == MP_ALLOC_ANON)
3099                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3100                                                      NULL);
3101                 }
3102         }
3103         if (ports != NULL) {
3104                 no_link_check = 1;
3105                 RTE_ETH_FOREACH_DEV(pt_id) {
3106                         printf("\nStopping port %d...\n", pt_id);
3107                         fflush(stdout);
3108                         stop_port(pt_id);
3109                 }
3110                 RTE_ETH_FOREACH_DEV(pt_id) {
3111                         printf("\nShutting down port %d...\n", pt_id);
3112                         fflush(stdout);
3113                         close_port(pt_id);
3114                 }
3115         }
3116
3117         if (hot_plug) {
3118                 ret = rte_dev_event_monitor_stop();
3119                 if (ret) {
3120                         RTE_LOG(ERR, EAL,
3121                                 "fail to stop device event monitor.");
3122                         return;
3123                 }
3124
3125                 ret = rte_dev_event_callback_unregister(NULL,
3126                         dev_event_callback, NULL);
3127                 if (ret < 0) {
3128                         RTE_LOG(ERR, EAL,
3129                                 "fail to unregister device event callback.\n");
3130                         return;
3131                 }
3132
3133                 ret = rte_dev_hotplug_handle_disable();
3134                 if (ret) {
3135                         RTE_LOG(ERR, EAL,
3136                                 "fail to disable hotplug handling.\n");
3137                         return;
3138                 }
3139         }
3140         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3141                 if (mempools[i])
3142                         rte_mempool_free(mempools[i]);
3143         }
3144
3145         printf("\nBye...\n");
3146 }
3147
3148 typedef void (*cmd_func_t)(void);
3149 struct pmd_test_command {
3150         const char *cmd_name;
3151         cmd_func_t cmd_func;
3152 };
3153
3154 /* Check the link status of all ports in up to 9s, and print them finally */
3155 static void
3156 check_all_ports_link_status(uint32_t port_mask)
3157 {
3158 #define CHECK_INTERVAL 100 /* 100ms */
3159 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3160         portid_t portid;
3161         uint8_t count, all_ports_up, print_flag = 0;
3162         struct rte_eth_link link;
3163         int ret;
3164         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3165
3166         printf("Checking link statuses...\n");
3167         fflush(stdout);
3168         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3169                 all_ports_up = 1;
3170                 RTE_ETH_FOREACH_DEV(portid) {
3171                         if ((port_mask & (1 << portid)) == 0)
3172                                 continue;
3173                         memset(&link, 0, sizeof(link));
3174                         ret = rte_eth_link_get_nowait(portid, &link);
3175                         if (ret < 0) {
3176                                 all_ports_up = 0;
3177                                 if (print_flag == 1)
3178                                         printf("Port %u link get failed: %s\n",
3179                                                 portid, rte_strerror(-ret));
3180                                 continue;
3181                         }
3182                         /* print link status if flag set */
3183                         if (print_flag == 1) {
3184                                 rte_eth_link_to_str(link_status,
3185                                         sizeof(link_status), &link);
3186                                 printf("Port %d %s\n", portid, link_status);
3187                                 continue;
3188                         }
3189                         /* clear all_ports_up flag if any link down */
3190                         if (link.link_status == ETH_LINK_DOWN) {
3191                                 all_ports_up = 0;
3192                                 break;
3193                         }
3194                 }
3195                 /* after finally printing all link status, get out */
3196                 if (print_flag == 1)
3197                         break;
3198
3199                 if (all_ports_up == 0) {
3200                         fflush(stdout);
3201                         rte_delay_ms(CHECK_INTERVAL);
3202                 }
3203
3204                 /* set the print_flag if all ports up or timeout */
3205                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3206                         print_flag = 1;
3207                 }
3208
3209                 if (lsc_interrupt)
3210                         break;
3211         }
3212 }
3213
3214 static void
3215 rmv_port_callback(void *arg)
3216 {
3217         int need_to_start = 0;
3218         int org_no_link_check = no_link_check;
3219         portid_t port_id = (intptr_t)arg;
3220         struct rte_device *dev;
3221
3222         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3223
3224         if (!test_done && port_is_forwarding(port_id)) {
3225                 need_to_start = 1;
3226                 stop_packet_forwarding();
3227         }
3228         no_link_check = 1;
3229         stop_port(port_id);
3230         no_link_check = org_no_link_check;
3231
3232         /* Save rte_device pointer before closing ethdev port */
3233         dev = rte_eth_devices[port_id].device;
3234         close_port(port_id);
3235         detach_device(dev); /* might be already removed or have more ports */
3236
3237         if (need_to_start)
3238                 start_packet_forwarding(0);
3239 }
3240
3241 /* This function is used by the interrupt thread */
3242 static int
3243 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3244                   void *ret_param)
3245 {
3246         RTE_SET_USED(param);
3247         RTE_SET_USED(ret_param);
3248
3249         if (type >= RTE_ETH_EVENT_MAX) {
3250                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3251                         port_id, __func__, type);
3252                 fflush(stderr);
3253         } else if (event_print_mask & (UINT32_C(1) << type)) {
3254                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3255                         eth_event_desc[type]);
3256                 fflush(stdout);
3257         }
3258
3259         switch (type) {
3260         case RTE_ETH_EVENT_NEW:
3261                 ports[port_id].need_setup = 1;
3262                 ports[port_id].port_status = RTE_PORT_HANDLING;
3263                 break;
3264         case RTE_ETH_EVENT_INTR_RMV:
3265                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3266                         break;
3267                 if (rte_eal_alarm_set(100000,
3268                                 rmv_port_callback, (void *)(intptr_t)port_id))
3269                         fprintf(stderr, "Could not set up deferred device removal\n");
3270                 break;
3271         case RTE_ETH_EVENT_DESTROY:
3272                 ports[port_id].port_status = RTE_PORT_CLOSED;
3273                 printf("Port %u is closed\n", port_id);
3274                 break;
3275         default:
3276                 break;
3277         }
3278         return 0;
3279 }
3280
3281 static int
3282 register_eth_event_callback(void)
3283 {
3284         int ret;
3285         enum rte_eth_event_type event;
3286
3287         for (event = RTE_ETH_EVENT_UNKNOWN;
3288                         event < RTE_ETH_EVENT_MAX; event++) {
3289                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3290                                 event,
3291                                 eth_event_callback,
3292                                 NULL);
3293                 if (ret != 0) {
3294                         TESTPMD_LOG(ERR, "Failed to register callback for "
3295                                         "%s event\n", eth_event_desc[event]);
3296                         return -1;
3297                 }
3298         }
3299
3300         return 0;
3301 }
3302
3303 /* This function is used by the interrupt thread */
3304 static void
3305 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3306                              __rte_unused void *arg)
3307 {
3308         uint16_t port_id;
3309         int ret;
3310
3311         if (type >= RTE_DEV_EVENT_MAX) {
3312                 fprintf(stderr, "%s called upon invalid event %d\n",
3313                         __func__, type);
3314                 fflush(stderr);
3315         }
3316
3317         switch (type) {
3318         case RTE_DEV_EVENT_REMOVE:
3319                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3320                         device_name);
3321                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3322                 if (ret) {
3323                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3324                                 device_name);
3325                         return;
3326                 }
3327                 /*
3328                  * Because the user's callback is invoked in eal interrupt
3329                  * callback, the interrupt callback need to be finished before
3330                  * it can be unregistered when detaching device. So finish
3331                  * callback soon and use a deferred removal to detach device
3332                  * is need. It is a workaround, once the device detaching be
3333                  * moved into the eal in the future, the deferred removal could
3334                  * be deleted.
3335                  */
3336                 if (rte_eal_alarm_set(100000,
3337                                 rmv_port_callback, (void *)(intptr_t)port_id))
3338                         RTE_LOG(ERR, EAL,
3339                                 "Could not set up deferred device removal\n");
3340                 break;
3341         case RTE_DEV_EVENT_ADD:
3342                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3343                         device_name);
3344                 /* TODO: After finish kernel driver binding,
3345                  * begin to attach port.
3346                  */
3347                 break;
3348         default:
3349                 break;
3350         }
3351 }
3352
3353 static int
3354 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3355 {
3356         uint16_t i;
3357         int diag;
3358         uint8_t mapping_found = 0;
3359
3360         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3361                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3362                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3363                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3364                                         tx_queue_stats_mappings[i].queue_id,
3365                                         tx_queue_stats_mappings[i].stats_counter_id);
3366                         if (diag != 0)
3367                                 return diag;
3368                         mapping_found = 1;
3369                 }
3370         }
3371         if (mapping_found)
3372                 port->tx_queue_stats_mapping_enabled = 1;
3373         return 0;
3374 }
3375
3376 static int
3377 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3378 {
3379         uint16_t i;
3380         int diag;
3381         uint8_t mapping_found = 0;
3382
3383         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3384                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3385                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3386                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3387                                         rx_queue_stats_mappings[i].queue_id,
3388                                         rx_queue_stats_mappings[i].stats_counter_id);
3389                         if (diag != 0)
3390                                 return diag;
3391                         mapping_found = 1;
3392                 }
3393         }
3394         if (mapping_found)
3395                 port->rx_queue_stats_mapping_enabled = 1;
3396         return 0;
3397 }
3398
3399 static void
3400 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3401 {
3402         int diag = 0;
3403
3404         diag = set_tx_queue_stats_mapping_registers(pi, port);
3405         if (diag != 0) {
3406                 if (diag == -ENOTSUP) {
3407                         port->tx_queue_stats_mapping_enabled = 0;
3408                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3409                 }
3410                 else
3411                         rte_exit(EXIT_FAILURE,
3412                                         "set_tx_queue_stats_mapping_registers "
3413                                         "failed for port id=%d diag=%d\n",
3414                                         pi, diag);
3415         }
3416
3417         diag = set_rx_queue_stats_mapping_registers(pi, port);
3418         if (diag != 0) {
3419                 if (diag == -ENOTSUP) {
3420                         port->rx_queue_stats_mapping_enabled = 0;
3421                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3422                 }
3423                 else
3424                         rte_exit(EXIT_FAILURE,
3425                                         "set_rx_queue_stats_mapping_registers "
3426                                         "failed for port id=%d diag=%d\n",
3427                                         pi, diag);
3428         }
3429 }
3430
3431 static void
3432 rxtx_port_config(struct rte_port *port)
3433 {
3434         uint16_t qid;
3435         uint64_t offloads;
3436
3437         for (qid = 0; qid < nb_rxq; qid++) {
3438                 offloads = port->rx_conf[qid].offloads;
3439                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3440                 if (offloads != 0)
3441                         port->rx_conf[qid].offloads = offloads;
3442
3443                 /* Check if any Rx parameters have been passed */
3444                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3445                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3446
3447                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3448                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3449
3450                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3451                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3452
3453                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3454                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3455
3456                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3457                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3458
3459                 port->nb_rx_desc[qid] = nb_rxd;
3460         }
3461
3462         for (qid = 0; qid < nb_txq; qid++) {
3463                 offloads = port->tx_conf[qid].offloads;
3464                 port->tx_conf[qid] = port->dev_info.default_txconf;
3465                 if (offloads != 0)
3466                         port->tx_conf[qid].offloads = offloads;
3467
3468                 /* Check if any Tx parameters have been passed */
3469                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3470                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3471
3472                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3473                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3474
3475                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3476                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3477
3478                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3479                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3480
3481                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3482                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3483
3484                 port->nb_tx_desc[qid] = nb_txd;
3485         }
3486 }
3487
3488 void
3489 init_port_config(void)
3490 {
3491         portid_t pid;
3492         struct rte_port *port;
3493         int ret;
3494
3495         RTE_ETH_FOREACH_DEV(pid) {
3496                 port = &ports[pid];
3497                 port->dev_conf.fdir_conf = fdir_conf;
3498
3499                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3500                 if (ret != 0)
3501                         return;
3502
3503                 if (nb_rxq > 1) {
3504                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3505                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3506                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3507                 } else {
3508                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3509                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3510                 }
3511
3512                 if (port->dcb_flag == 0) {
3513                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3514                                 port->dev_conf.rxmode.mq_mode =
3515                                         (enum rte_eth_rx_mq_mode)
3516                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3517                         else
3518                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3519                 }
3520
3521                 rxtx_port_config(port);
3522
3523                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3524                 if (ret != 0)
3525                         return;
3526
3527                 map_port_queue_stats_mapping_registers(pid, port);
3528 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3529                 rte_pmd_ixgbe_bypass_init(pid);
3530 #endif
3531
3532                 if (lsc_interrupt &&
3533                     (rte_eth_devices[pid].data->dev_flags &
3534                      RTE_ETH_DEV_INTR_LSC))
3535                         port->dev_conf.intr_conf.lsc = 1;
3536                 if (rmv_interrupt &&
3537                     (rte_eth_devices[pid].data->dev_flags &
3538                      RTE_ETH_DEV_INTR_RMV))
3539                         port->dev_conf.intr_conf.rmv = 1;
3540         }
3541 }
3542
3543 void set_port_slave_flag(portid_t slave_pid)
3544 {
3545         struct rte_port *port;
3546
3547         port = &ports[slave_pid];
3548         port->slave_flag = 1;
3549 }
3550
3551 void clear_port_slave_flag(portid_t slave_pid)
3552 {
3553         struct rte_port *port;
3554
3555         port = &ports[slave_pid];
3556         port->slave_flag = 0;
3557 }
3558
3559 uint8_t port_is_bonding_slave(portid_t slave_pid)
3560 {
3561         struct rte_port *port;
3562
3563         port = &ports[slave_pid];
3564         if ((rte_eth_devices[slave_pid].data->dev_flags &
3565             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3566                 return 1;
3567         return 0;
3568 }
3569
3570 const uint16_t vlan_tags[] = {
3571                 0,  1,  2,  3,  4,  5,  6,  7,
3572                 8,  9, 10, 11,  12, 13, 14, 15,
3573                 16, 17, 18, 19, 20, 21, 22, 23,
3574                 24, 25, 26, 27, 28, 29, 30, 31
3575 };
3576
3577 static  int
3578 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3579                  enum dcb_mode_enable dcb_mode,
3580                  enum rte_eth_nb_tcs num_tcs,
3581                  uint8_t pfc_en)
3582 {
3583         uint8_t i;
3584         int32_t rc;
3585         struct rte_eth_rss_conf rss_conf;
3586
3587         /*
3588          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3589          * given above, and the number of traffic classes available for use.
3590          */
3591         if (dcb_mode == DCB_VT_ENABLED) {
3592                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3593                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3594                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3595                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3596
3597                 /* VMDQ+DCB RX and TX configurations */
3598                 vmdq_rx_conf->enable_default_pool = 0;
3599                 vmdq_rx_conf->default_pool = 0;
3600                 vmdq_rx_conf->nb_queue_pools =
3601                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3602                 vmdq_tx_conf->nb_queue_pools =
3603                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3604
3605                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3606                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3607                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3608                         vmdq_rx_conf->pool_map[i].pools =
3609                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3610                 }
3611                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3612                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3613                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3614                 }
3615
3616                 /* set DCB mode of RX and TX of multiple queues */
3617                 eth_conf->rxmode.mq_mode =
3618                                 (enum rte_eth_rx_mq_mode)
3619                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3620                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3621         } else {
3622                 struct rte_eth_dcb_rx_conf *rx_conf =
3623                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3624                 struct rte_eth_dcb_tx_conf *tx_conf =
3625                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3626
3627                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3628
3629                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3630                 if (rc != 0)
3631                         return rc;
3632
3633                 rx_conf->nb_tcs = num_tcs;
3634                 tx_conf->nb_tcs = num_tcs;
3635
3636                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3637                         rx_conf->dcb_tc[i] = i % num_tcs;
3638                         tx_conf->dcb_tc[i] = i % num_tcs;
3639                 }
3640
3641                 eth_conf->rxmode.mq_mode =
3642                                 (enum rte_eth_rx_mq_mode)
3643                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3644                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3645                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3646         }
3647
3648         if (pfc_en)
3649                 eth_conf->dcb_capability_en =
3650                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3651         else
3652                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3653
3654         return 0;
3655 }
3656
3657 int
3658 init_port_dcb_config(portid_t pid,
3659                      enum dcb_mode_enable dcb_mode,
3660                      enum rte_eth_nb_tcs num_tcs,
3661                      uint8_t pfc_en)
3662 {
3663         struct rte_eth_conf port_conf;
3664         struct rte_port *rte_port;
3665         int retval;
3666         uint16_t i;
3667
3668         rte_port = &ports[pid];
3669
3670         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3671         /* Enter DCB configuration status */
3672         dcb_config = 1;
3673
3674         port_conf.rxmode = rte_port->dev_conf.rxmode;
3675         port_conf.txmode = rte_port->dev_conf.txmode;
3676
3677         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3678         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3679         if (retval < 0)
3680                 return retval;
3681         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3682
3683         /* re-configure the device . */
3684         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3685         if (retval < 0)
3686                 return retval;
3687
3688         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3689         if (retval != 0)
3690                 return retval;
3691
3692         /* If dev_info.vmdq_pool_base is greater than 0,
3693          * the queue id of vmdq pools is started after pf queues.
3694          */
3695         if (dcb_mode == DCB_VT_ENABLED &&
3696             rte_port->dev_info.vmdq_pool_base > 0) {
3697                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3698                         " for port %d.", pid);
3699                 return -1;
3700         }
3701
3702         /* Assume the ports in testpmd have the same dcb capability
3703          * and has the same number of rxq and txq in dcb mode
3704          */
3705         if (dcb_mode == DCB_VT_ENABLED) {
3706                 if (rte_port->dev_info.max_vfs > 0) {
3707                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3708                         nb_txq = rte_port->dev_info.nb_tx_queues;
3709                 } else {
3710                         nb_rxq = rte_port->dev_info.max_rx_queues;
3711                         nb_txq = rte_port->dev_info.max_tx_queues;
3712                 }
3713         } else {
3714                 /*if vt is disabled, use all pf queues */
3715                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3716                         nb_rxq = rte_port->dev_info.max_rx_queues;
3717                         nb_txq = rte_port->dev_info.max_tx_queues;
3718                 } else {
3719                         nb_rxq = (queueid_t)num_tcs;
3720                         nb_txq = (queueid_t)num_tcs;
3721
3722                 }
3723         }
3724         rx_free_thresh = 64;
3725
3726         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3727
3728         rxtx_port_config(rte_port);
3729         /* VLAN filter */
3730         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3731         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3732                 rx_vft_set(pid, vlan_tags[i], 1);
3733
3734         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3735         if (retval != 0)
3736                 return retval;
3737
3738         map_port_queue_stats_mapping_registers(pid, rte_port);
3739
3740         rte_port->dcb_flag = 1;
3741
3742         return 0;
3743 }
3744
3745 static void
3746 init_port(void)
3747 {
3748         int i;
3749
3750         /* Configuration of Ethernet ports. */
3751         ports = rte_zmalloc("testpmd: ports",
3752                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3753                             RTE_CACHE_LINE_SIZE);
3754         if (ports == NULL) {
3755                 rte_exit(EXIT_FAILURE,
3756                                 "rte_zmalloc(%d struct rte_port) failed\n",
3757                                 RTE_MAX_ETHPORTS);
3758         }
3759         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3760                 LIST_INIT(&ports[i].flow_tunnel_list);
3761         /* Initialize ports NUMA structures */
3762         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3763         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3764         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3765 }
3766
3767 static void
3768 force_quit(void)
3769 {
3770         pmd_test_exit();
3771         prompt_exit();
3772 }
3773
3774 static void
3775 print_stats(void)
3776 {
3777         uint8_t i;
3778         const char clr[] = { 27, '[', '2', 'J', '\0' };
3779         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3780
3781         /* Clear screen and move to top left */
3782         printf("%s%s", clr, top_left);
3783
3784         printf("\nPort statistics ====================================");
3785         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3786                 nic_stats_display(fwd_ports_ids[i]);
3787
3788         fflush(stdout);
3789 }
3790
3791 static void
3792 signal_handler(int signum)
3793 {
3794         if (signum == SIGINT || signum == SIGTERM) {
3795                 printf("\nSignal %d received, preparing to exit...\n",
3796                                 signum);
3797 #ifdef RTE_LIBRTE_PDUMP
3798                 /* uninitialize packet capture framework */
3799                 rte_pdump_uninit();
3800 #endif
3801 #ifdef RTE_LIBRTE_LATENCY_STATS
3802                 if (latencystats_enabled != 0)
3803                         rte_latencystats_uninit();
3804 #endif
3805                 force_quit();
3806                 /* Set flag to indicate the force termination. */
3807                 f_quit = 1;
3808                 /* exit with the expected status */
3809                 signal(signum, SIG_DFL);
3810                 kill(getpid(), signum);
3811         }
3812 }
3813
3814 int
3815 main(int argc, char** argv)
3816 {
3817         int diag;
3818         portid_t port_id;
3819         uint16_t count;
3820         int ret;
3821
3822         signal(SIGINT, signal_handler);
3823         signal(SIGTERM, signal_handler);
3824
3825         testpmd_logtype = rte_log_register("testpmd");
3826         if (testpmd_logtype < 0)
3827                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3828         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3829
3830         diag = rte_eal_init(argc, argv);
3831         if (diag < 0)
3832                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3833                          rte_strerror(rte_errno));
3834
3835         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3836                 rte_exit(EXIT_FAILURE,
3837                          "Secondary process type not supported.\n");
3838
3839         ret = register_eth_event_callback();
3840         if (ret != 0)
3841                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3842
3843 #ifdef RTE_LIBRTE_PDUMP
3844         /* initialize packet capture framework */
3845         rte_pdump_init();
3846 #endif
3847
3848         count = 0;
3849         RTE_ETH_FOREACH_DEV(port_id) {
3850                 ports_ids[count] = port_id;
3851                 count++;
3852         }
3853         nb_ports = (portid_t) count;
3854         if (nb_ports == 0)
3855                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3856
3857         /* allocate port structures, and init them */
3858         init_port();
3859
3860         set_def_fwd_config();
3861         if (nb_lcores == 0)
3862                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3863                          "Check the core mask argument\n");
3864
3865         /* Bitrate/latency stats disabled by default */
3866 #ifdef RTE_LIBRTE_BITRATESTATS
3867         bitrate_enabled = 0;
3868 #endif
3869 #ifdef RTE_LIBRTE_LATENCY_STATS
3870         latencystats_enabled = 0;
3871 #endif
3872
3873         /* on FreeBSD, mlockall() is disabled by default */
3874 #ifdef RTE_EXEC_ENV_FREEBSD
3875         do_mlockall = 0;
3876 #else
3877         do_mlockall = 1;
3878 #endif
3879
3880         argc -= diag;
3881         argv += diag;
3882         if (argc > 1)
3883                 launch_args_parse(argc, argv);
3884
3885         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3886                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3887                         strerror(errno));
3888         }
3889
3890         if (tx_first && interactive)
3891                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3892                                 "interactive mode.\n");
3893
3894         if (tx_first && lsc_interrupt) {
3895                 printf("Warning: lsc_interrupt needs to be off when "
3896                                 " using tx_first. Disabling.\n");
3897                 lsc_interrupt = 0;
3898         }
3899
3900         if (!nb_rxq && !nb_txq)
3901                 printf("Warning: Either rx or tx queues should be non-zero\n");
3902
3903         if (nb_rxq > 1 && nb_rxq > nb_txq)
3904                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3905                        "but nb_txq=%d will prevent to fully test it.\n",
3906                        nb_rxq, nb_txq);
3907
3908         init_config();
3909
3910         if (hot_plug) {
3911                 ret = rte_dev_hotplug_handle_enable();
3912                 if (ret) {
3913                         RTE_LOG(ERR, EAL,
3914                                 "fail to enable hotplug handling.");
3915                         return -1;
3916                 }
3917
3918                 ret = rte_dev_event_monitor_start();
3919                 if (ret) {
3920                         RTE_LOG(ERR, EAL,
3921                                 "fail to start device event monitoring.");
3922                         return -1;
3923                 }
3924
3925                 ret = rte_dev_event_callback_register(NULL,
3926                         dev_event_callback, NULL);
3927                 if (ret) {
3928                         RTE_LOG(ERR, EAL,
3929                                 "fail  to register device event callback\n");
3930                         return -1;
3931                 }
3932         }
3933
3934         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3935                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3936
3937         /* set all ports to promiscuous mode by default */
3938         RTE_ETH_FOREACH_DEV(port_id) {
3939                 ret = rte_eth_promiscuous_enable(port_id);
3940                 if (ret != 0)
3941                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3942                                 port_id, rte_strerror(-ret));
3943         }
3944
3945         /* Init metrics library */
3946         rte_metrics_init(rte_socket_id());
3947
3948 #ifdef RTE_LIBRTE_LATENCY_STATS
3949         if (latencystats_enabled != 0) {
3950                 int ret = rte_latencystats_init(1, NULL);
3951                 if (ret)
3952                         printf("Warning: latencystats init()"
3953                                 " returned error %d\n", ret);
3954                 printf("Latencystats running on lcore %d\n",
3955                         latencystats_lcore_id);
3956         }
3957 #endif
3958
3959         /* Setup bitrate stats */
3960 #ifdef RTE_LIBRTE_BITRATESTATS
3961         if (bitrate_enabled != 0) {
3962                 bitrate_data = rte_stats_bitrate_create();
3963                 if (bitrate_data == NULL)
3964                         rte_exit(EXIT_FAILURE,
3965                                 "Could not allocate bitrate data.\n");
3966                 rte_stats_bitrate_reg(bitrate_data);
3967         }
3968 #endif
3969
3970 #ifdef RTE_LIBRTE_CMDLINE
3971         if (strlen(cmdline_filename) != 0)
3972                 cmdline_read_from_file(cmdline_filename);
3973
3974         if (interactive == 1) {
3975                 if (auto_start) {
3976                         printf("Start automatic packet forwarding\n");
3977                         start_packet_forwarding(0);
3978                 }
3979                 prompt();
3980                 pmd_test_exit();
3981         } else
3982 #endif
3983         {
3984                 char c;
3985                 int rc;
3986
3987                 f_quit = 0;
3988
3989                 printf("No commandline core given, start packet forwarding\n");
3990                 start_packet_forwarding(tx_first);
3991                 if (stats_period != 0) {
3992                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3993                         uint64_t timer_period;
3994
3995                         /* Convert to number of cycles */
3996                         timer_period = stats_period * rte_get_timer_hz();
3997
3998                         while (f_quit == 0) {
3999                                 cur_time = rte_get_timer_cycles();
4000                                 diff_time += cur_time - prev_time;
4001
4002                                 if (diff_time >= timer_period) {
4003                                         print_stats();
4004                                         /* Reset the timer */
4005                                         diff_time = 0;
4006                                 }
4007                                 /* Sleep to avoid unnecessary checks */
4008                                 prev_time = cur_time;
4009                                 sleep(1);
4010                         }
4011                 }
4012
4013                 printf("Press enter to exit\n");
4014                 rc = read(0, &c, 1);
4015                 pmd_test_exit();
4016                 if (rc < 0)
4017                         return 1;
4018         }
4019
4020         ret = rte_eal_cleanup();
4021         if (ret != 0)
4022                 rte_exit(EXIT_FAILURE,
4023                          "EAL cleanup failed: %s\n", strerror(-ret));
4024
4025         return EXIT_SUCCESS;
4026 }