app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #ifndef RTE_EXEC_ENV_WINDOWS
  13 #include <sys/mman.h>
  14 #endif
  15 #include <sys/types.h>
  16 #include <errno.h>
  17 #include <stdbool.h>
  18
  19 #include <sys/queue.h>
  20 #include <sys/stat.h>
  21
  22 #include <stdint.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_byteorder.h>
  29 #include <rte_log.h>
  30 #include <rte_debug.h>
  31 #include <rte_cycles.h>
  32 #include <rte_memory.h>
  33 #include <rte_memcpy.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_alarm.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_atomic.h>
  40 #include <rte_branch_prediction.h>
  41 #include <rte_mempool.h>
  42 #include <rte_malloc.h>
  43 #include <rte_mbuf.h>
  44 #include <rte_mbuf_pool_ops.h>
  45 #include <rte_interrupts.h>
  46 #include <rte_pci.h>
  47 #include <rte_ether.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_dev.h>
  50 #include <rte_string_fns.h>
  51 #ifdef RTE_NET_IXGBE
  52 #include <rte_pmd_ixgbe.h>
  53 #endif
  54 #ifdef RTE_LIB_PDUMP
  55 #include <rte_pdump.h>
  56 #endif
  57 #include <rte_flow.h>
  58 #include <rte_metrics.h>
  59 #ifdef RTE_LIB_BITRATESTATS
  60 #include <rte_bitrate.h>
  61 #endif
  62 #ifdef RTE_LIB_LATENCYSTATS
  63 #include <rte_latencystats.h>
  64 #endif
  65 #ifdef RTE_EXEC_ENV_WINDOWS
  66 #include <process.h>
  67 #endif
  68
  69 #include "testpmd.h"
  70
  71 #ifndef MAP_HUGETLB
  72 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  73 #define HUGE_FLAG (0x40000)
  74 #else
  75 #define HUGE_FLAG MAP_HUGETLB
  76 #endif
  77
  78 #ifndef MAP_HUGE_SHIFT
  79 /* older kernels (or FreeBSD) will not have this define */
  80 #define HUGE_SHIFT (26)
  81 #else
  82 #define HUGE_SHIFT MAP_HUGE_SHIFT
  83 #endif
  84
  85 #define EXTMEM_HEAP_NAME "extmem"
  86 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  87
  88 uint16_t verbose_level = 0; /**< Silent by default. */
  89 int testpmd_logtype; /**< Log type for testpmd logs */
  90
  91 /* use main core for command line ? */
  92 uint8_t interactive = 0;
  93 uint8_t auto_start = 0;
  94 uint8_t tx_first;
  95 char cmdline_filename[PATH_MAX] = {0};
  96
  97 /*
  98  * NUMA support configuration.
  99  * When set, the NUMA support attempts to dispatch the allocation of the
 100  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 101  * probed ports among the CPU sockets 0 and 1.
 102  * Otherwise, all memory is allocated from CPU socket 0.
 103  */
 104 uint8_t numa_support = 1; /**< numa enabled by default */
 105
 106 /*
 107  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 108  * not configured.
 109  */
 110 uint8_t socket_num = UMA_NO_CONFIG;
 111
 112 /*
 113  * Select mempool allocation type:
 114  * - native: use regular DPDK memory
 115  * - anon: use regular DPDK memory to create mempool, but populate using
 116  *         anonymous memory (may not be IOVA-contiguous)
 117  * - xmem: use externally allocated hugepage memory
 118  */
 119 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 120
 121 /*
 122  * Store specified sockets on which memory pool to be used by ports
 123  * is allocated.
 124  */
 125 uint8_t port_numa[RTE_MAX_ETHPORTS];
 126
 127 /*
 128  * Store specified sockets on which RX ring to be used by ports
 129  * is allocated.
 130  */
 131 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 132
 133 /*
 134  * Store specified sockets on which TX ring to be used by ports
 135  * is allocated.
 136  */
 137 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 138
 139 /*
 140  * Record the Ethernet address of peer target ports to which packets are
 141  * forwarded.
 142  * Must be instantiated with the ethernet addresses of peer traffic generator
 143  * ports.
 144  */
 145 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 146 portid_t nb_peer_eth_addrs = 0;
 147
 148 /*
 149  * Probed Target Environment.
 150  */
 151 struct rte_port *ports;        /**< For all probed ethernet ports. */
 152 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 153 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 154 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 155
 156 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 157
 158 /*
 159  * Test Forwarding Configuration.
 160  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 161  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 162  */
 163 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 164 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 165 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 166 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 167
 168 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 169 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 170
 171 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 172 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 173
 174 /*
 175  * Forwarding engines.
 176  */
 177 struct fwd_engine * fwd_engines[] = {
 178         &io_fwd_engine,
 179         &mac_fwd_engine,
 180         &mac_swap_engine,
 181         &flow_gen_engine,
 182         &rx_only_engine,
 183         &tx_only_engine,
 184         &csum_fwd_engine,
 185         &icmp_echo_engine,
 186         &noisy_vnf_engine,
 187         &five_tuple_swap_fwd_engine,
 188 #ifdef RTE_LIBRTE_IEEE1588
 189         &ieee1588_fwd_engine,
 190 #endif
 191         NULL,
 192 };
 193
 194 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 195 uint16_t mempool_flags;
 196
 197 struct fwd_config cur_fwd_config;
 198 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 199 uint32_t retry_enabled;
 200 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 201 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 202
 203 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 204 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 205         DEFAULT_MBUF_DATA_SIZE
 206 }; /**< Mbuf data space size. */
 207 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 208                                       * specified on command-line. */
 209 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 210
 211 /*
 212  * In container, it cannot terminate the process which running with 'stats-period'
 213  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 214  */
 215 uint8_t f_quit;
 216
 217 /*
 218  * Configuration of packet segments used to scatter received packets
 219  * if some of split features is configured.
 220  */
 221 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 222 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 223 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 224 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 225
 226 /*
 227  * Configuration of packet segments used by the "txonly" processing engine.
 228  */
 229 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 230 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 231         TXONLY_DEF_PACKET_LEN,
 232 };
 233 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 234
 235 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 236 /**< Split policy for packets to TX. */
 237
 238 uint8_t txonly_multi_flow;
 239 /**< Whether multiple flows are generated in TXONLY mode. */
 240
 241 uint32_t tx_pkt_times_inter;
 242 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 243
 244 uint32_t tx_pkt_times_intra;
 245 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 246
 247 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 248 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 249 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */
 250 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 251
 252 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 253 uint8_t dcb_config = 0;
 254
 255 /*
 256  * Configurable number of RX/TX queues.
 257  */
 258 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 259 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 260 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 261
 262 /*
 263  * Configurable number of RX/TX ring descriptors.
 264  * Defaults are supplied by drivers via ethdev.
 265  */
 266 #define RTE_TEST_RX_DESC_DEFAULT 0
 267 #define RTE_TEST_TX_DESC_DEFAULT 0
 268 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 269 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 270
 271 #define RTE_PMD_PARAM_UNSET -1
 272 /*
 273  * Configurable values of RX and TX ring threshold registers.
 274  */
 275
 276 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 277 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 278 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 279
 280 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 281 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 282 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 283
 284 /*
 285  * Configurable value of RX free threshold.
 286  */
 287 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 288
 289 /*
 290  * Configurable value of RX drop enable.
 291  */
 292 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 293
 294 /*
 295  * Configurable value of TX free threshold.
 296  */
 297 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 298
 299 /*
 300  * Configurable value of TX RS bit threshold.
 301  */
 302 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 303
 304 /*
 305  * Configurable value of buffered packets before sending.
 306  */
 307 uint16_t noisy_tx_sw_bufsz;
 308
 309 /*
 310  * Configurable value of packet buffer timeout.
 311  */
 312 uint16_t noisy_tx_sw_buf_flush_time;
 313
 314 /*
 315  * Configurable value for size of VNF internal memory area
 316  * used for simulating noisy neighbour behaviour
 317  */
 318 uint64_t noisy_lkup_mem_sz;
 319
 320 /*
 321  * Configurable value of number of random writes done in
 322  * VNF simulation memory area.
 323  */
 324 uint64_t noisy_lkup_num_writes;
 325
 326 /*
 327  * Configurable value of number of random reads done in
 328  * VNF simulation memory area.
 329  */
 330 uint64_t noisy_lkup_num_reads;
 331
 332 /*
 333  * Configurable value of number of random reads/writes done in
 334  * VNF simulation memory area.
 335  */
 336 uint64_t noisy_lkup_num_reads_writes;
 337
 338 /*
 339  * Receive Side Scaling (RSS) configuration.
 340  */
 341 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 342
 343 /*
 344  * Port topology configuration
 345  */
 346 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 347
 348 /*
 349  * Avoids to flush all the RX streams before starts forwarding.
 350  */
 351 uint8_t no_flush_rx = 0; /* flush by default */
 352
 353 /*
 354  * Flow API isolated mode.
 355  */
 356 uint8_t flow_isolate_all;
 357
 358 /*
 359  * Avoids to check link status when starting/stopping a port.
 360  */
 361 uint8_t no_link_check = 0; /* check by default */
 362
 363 /*
 364  * Don't automatically start all ports in interactive mode.
 365  */
 366 uint8_t no_device_start = 0;
 367
 368 /*
 369  * Enable link status change notification
 370  */
 371 uint8_t lsc_interrupt = 1; /* enabled by default */
 372
 373 /*
 374  * Enable device removal notification.
 375  */
 376 uint8_t rmv_interrupt = 1; /* enabled by default */
 377
 378 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 379
 380 /* After attach, port setup is called on event or by iterator */
 381 bool setup_on_probe_event = true;
 382
 383 /* Clear ptypes on port initialization. */
 384 uint8_t clear_ptypes = true;
 385
 386 /* Hairpin ports configuration mode. */
 387 uint16_t hairpin_mode;
 388
 389 /* Pretty printing of ethdev events */
 390 static const char * const eth_event_desc[] = {
 391         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 392         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 393         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 394         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 395         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 396         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 397         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 398         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 399         [RTE_ETH_EVENT_NEW] = "device probed",
 400         [RTE_ETH_EVENT_DESTROY] = "device released",
 401         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 402         [RTE_ETH_EVENT_MAX] = NULL,
 403 };
 404
 405 /*
 406  * Display or mask ether events
 407  * Default to all events except VF_MBOX
 408  */
 409 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 413                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 414                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 415                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 416                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 417 /*
 418  * Decide if all memory are locked for performance.
 419  */
 420 int do_mlockall = 0;
 421
 422 /*
 423  * NIC bypass mode configuration options.
 424  */
 425
 426 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 427 /* The NIC bypass watchdog timeout. */
 428 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 429 #endif
 430
 431
 432 #ifdef RTE_LIB_LATENCYSTATS
 433
 434 /*
 435  * Set when latency stats is enabled in the commandline
 436  */
 437 uint8_t latencystats_enabled;
 438
 439 /*
 440  * Lcore ID to serive latency statistics.
 441  */
 442 lcoreid_t latencystats_lcore_id = -1;
 443
 444 #endif
 445
 446 /*
 447  * Ethernet device configuration.
 448  */
 449 struct rte_eth_rxmode rx_mode = {
 450         /* Default maximum frame length.
 451          * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
 452          * in init_config().
 453          */
 454         .max_rx_pkt_len = 0,
 455 };
 456
 457 struct rte_eth_txmode tx_mode = {
 458         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 459 };
 460
 461 struct rte_fdir_conf fdir_conf = {
 462         .mode = RTE_FDIR_MODE_NONE,
 463         .pballoc = RTE_FDIR_PBALLOC_64K,
 464         .status = RTE_FDIR_REPORT_STATUS,
 465         .mask = {
 466                 .vlan_tci_mask = 0xFFEF,
 467                 .ipv4_mask     = {
 468                         .src_ip = 0xFFFFFFFF,
 469                         .dst_ip = 0xFFFFFFFF,
 470                 },
 471                 .ipv6_mask     = {
 472                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 473                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 474                 },
 475                 .src_port_mask = 0xFFFF,
 476                 .dst_port_mask = 0xFFFF,
 477                 .mac_addr_byte_mask = 0xFF,
 478                 .tunnel_type_mask = 1,
 479                 .tunnel_id_mask = 0xFFFFFFFF,
 480         },
 481         .drop_queue = 127,
 482 };
 483
 484 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 485
 486 /*
 487  * Display zero values by default for xstats
 488  */
 489 uint8_t xstats_hide_zero;
 490
 491 /*
 492  * Measure of CPU cycles disabled by default
 493  */
 494 uint8_t record_core_cycles;
 495
 496 /*
 497  * Display of RX and TX bursts disabled by default
 498  */
 499 uint8_t record_burst_stats;
 500
 501 unsigned int num_sockets = 0;
 502 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 503
 504 #ifdef RTE_LIB_BITRATESTATS
 505 /* Bitrate statistics */
 506 struct rte_stats_bitrates *bitrate_data;
 507 lcoreid_t bitrate_lcore_id;
 508 uint8_t bitrate_enabled;
 509 #endif
 510
 511 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 512 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 513
 514 /*
 515  * hexadecimal bitmask of RX mq mode can be enabled.
 516  */
 517 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 518
 519 /*
 520  * Used to set forced link speed
 521  */
 522 uint32_t eth_link_speed;
 523
 524 /*
 525  * ID of the current process in multi-process, used to
 526  * configure the queues to be polled.
 527  */
 528 int proc_id;
 529
 530 /*
 531  * Number of processes in multi-process, used to
 532  * configure the queues to be polled.
 533  */
 534 unsigned int num_procs = 1;
 535
 536 static void
 537 eth_rx_metadata_negotiate_mp(uint16_t port_id)
 538 {
 539         uint64_t rx_meta_features = 0;
 540         int ret;
 541
 542         if (!is_proc_primary())
 543                 return;
 544
 545         rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG;
 546         rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK;
 547         rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID;
 548
 549         ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features);
 550         if (ret == 0) {
 551                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) {
 552                         TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n",
 553                                     port_id);
 554                 }
 555
 556                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) {
 557                         TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n",
 558                                     port_id);
 559                 }
 560
 561                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) {
 562                         TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n",
 563                                     port_id);
 564                 }
 565         } else if (ret != -ENOTSUP) {
 566                 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n",
 567                          port_id, rte_strerror(-ret));
 568         }
 569 }
 570
 571 static void
 572 flow_pick_transfer_proxy_mp(uint16_t port_id)
 573 {
 574         struct rte_port *port = &ports[port_id];
 575         int ret;
 576
 577         port->flow_transfer_proxy = port_id;
 578
 579         if (!is_proc_primary())
 580                 return;
 581
 582         ret = rte_flow_pick_transfer_proxy(port_id, &port->flow_transfer_proxy,
 583                                            NULL);
 584         if (ret != 0) {
 585                 fprintf(stderr, "Error picking flow transfer proxy for port %u: %s - ignore\n",
 586                         port_id, rte_strerror(-ret));
 587         }
 588 }
 589
 590 static int
 591 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 592                       const struct rte_eth_conf *dev_conf)
 593 {
 594         if (is_proc_primary())
 595                 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
 596                                         dev_conf);
 597         return 0;
 598 }
 599
 600 static int
 601 eth_dev_start_mp(uint16_t port_id)
 602 {
 603         if (is_proc_primary())
 604                 return rte_eth_dev_start(port_id);
 605
 606         return 0;
 607 }
 608
 609 static int
 610 eth_dev_stop_mp(uint16_t port_id)
 611 {
 612         if (is_proc_primary())
 613                 return rte_eth_dev_stop(port_id);
 614
 615         return 0;
 616 }
 617
 618 static void
 619 mempool_free_mp(struct rte_mempool *mp)
 620 {
 621         if (is_proc_primary())
 622                 rte_mempool_free(mp);
 623 }
 624
 625 static int
 626 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
 627 {
 628         if (is_proc_primary())
 629                 return rte_eth_dev_set_mtu(port_id, mtu);
 630
 631         return 0;
 632 }
 633
 634 /* Forward function declarations */
 635 static void setup_attached_port(portid_t pi);
 636 static void check_all_ports_link_status(uint32_t port_mask);
 637 static int eth_event_callback(portid_t port_id,
 638                               enum rte_eth_event_type type,
 639                               void *param, void *ret_param);
 640 static void dev_event_callback(const char *device_name,
 641                                 enum rte_dev_event_type type,
 642                                 void *param);
 643
 644 /*
 645  * Check if all the ports are started.
 646  * If yes, return positive value. If not, return zero.
 647  */
 648 static int all_ports_started(void);
 649
 650 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 651 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 652
 653 /* Holds the registered mbuf dynamic flags names. */
 654 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 655
 656 /*
 657  * Helper function to check if socket is already discovered.
 658  * If yes, return positive value. If not, return zero.
 659  */
 660 int
 661 new_socket_id(unsigned int socket_id)
 662 {
 663         unsigned int i;
 664
 665         for (i = 0; i < num_sockets; i++) {
 666                 if (socket_ids[i] == socket_id)
 667                         return 0;
 668         }
 669         return 1;
 670 }
 671
 672 /*
 673  * Setup default configuration.
 674  */
 675 static void
 676 set_default_fwd_lcores_config(void)
 677 {
 678         unsigned int i;
 679         unsigned int nb_lc;
 680         unsigned int sock_num;
 681
 682         nb_lc = 0;
 683         for (i = 0; i < RTE_MAX_LCORE; i++) {
 684                 if (!rte_lcore_is_enabled(i))
 685                         continue;
 686                 sock_num = rte_lcore_to_socket_id(i);
 687                 if (new_socket_id(sock_num)) {
 688                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 689                                 rte_exit(EXIT_FAILURE,
 690                                          "Total sockets greater than %u\n",
 691                                          RTE_MAX_NUMA_NODES);
 692                         }
 693                         socket_ids[num_sockets++] = sock_num;
 694                 }
 695                 if (i == rte_get_main_lcore())
 696                         continue;
 697                 fwd_lcores_cpuids[nb_lc++] = i;
 698         }
 699         nb_lcores = (lcoreid_t) nb_lc;
 700         nb_cfg_lcores = nb_lcores;
 701         nb_fwd_lcores = 1;
 702 }
 703
 704 static void
 705 set_def_peer_eth_addrs(void)
 706 {
 707         portid_t i;
 708
 709         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 710                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 711                 peer_eth_addrs[i].addr_bytes[5] = i;
 712         }
 713 }
 714
 715 static void
 716 set_default_fwd_ports_config(void)
 717 {
 718         portid_t pt_id;
 719         int i = 0;
 720
 721         RTE_ETH_FOREACH_DEV(pt_id) {
 722                 fwd_ports_ids[i++] = pt_id;
 723
 724                 /* Update sockets info according to the attached device */
 725                 int socket_id = rte_eth_dev_socket_id(pt_id);
 726                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 727                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 728                                 rte_exit(EXIT_FAILURE,
 729                                          "Total sockets greater than %u\n",
 730                                          RTE_MAX_NUMA_NODES);
 731                         }
 732                         socket_ids[num_sockets++] = socket_id;
 733                 }
 734         }
 735
 736         nb_cfg_ports = nb_ports;
 737         nb_fwd_ports = nb_ports;
 738 }
 739
 740 void
 741 set_def_fwd_config(void)
 742 {
 743         set_default_fwd_lcores_config();
 744         set_def_peer_eth_addrs();
 745         set_default_fwd_ports_config();
 746 }
 747
 748 #ifndef RTE_EXEC_ENV_WINDOWS
 749 /* extremely pessimistic estimation of memory required to create a mempool */
 750 static int
 751 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 752 {
 753         unsigned int n_pages, mbuf_per_pg, leftover;
 754         uint64_t total_mem, mbuf_mem, obj_sz;
 755
 756         /* there is no good way to predict how much space the mempool will
 757          * occupy because it will allocate chunks on the fly, and some of those
 758          * will come from default DPDK memory while some will come from our
 759          * external memory, so just assume 128MB will be enough for everyone.
 760          */
 761         uint64_t hdr_mem = 128 << 20;
 762
 763         /* account for possible non-contiguousness */
 764         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 765         if (obj_sz > pgsz) {
 766                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 767                 return -1;
 768         }
 769
 770         mbuf_per_pg = pgsz / obj_sz;
 771         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 772         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 773
 774         mbuf_mem = n_pages * pgsz;
 775
 776         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 777
 778         if (total_mem > SIZE_MAX) {
 779                 TESTPMD_LOG(ERR, "Memory size too big\n");
 780                 return -1;
 781         }
 782         *out = (size_t)total_mem;
 783
 784         return 0;
 785 }
 786
 787 static int
 788 pagesz_flags(uint64_t page_sz)
 789 {
 790         /* as per mmap() manpage, all page sizes are log2 of page size
 791          * shifted by MAP_HUGE_SHIFT
 792          */
 793         int log2 = rte_log2_u64(page_sz);
 794
 795         return (log2 << HUGE_SHIFT);
 796 }
 797
 798 static void *
 799 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 800 {
 801         void *addr;
 802         int flags;
 803
 804         /* allocate anonymous hugepages */
 805         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 806         if (huge)
 807                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 808
 809         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 810         if (addr == MAP_FAILED)
 811                 return NULL;
 812
 813         return addr;
 814 }
 815
 816 struct extmem_param {
 817         void *addr;
 818         size_t len;
 819         size_t pgsz;
 820         rte_iova_t *iova_table;
 821         unsigned int iova_table_len;
 822 };
 823
 824 static int
 825 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 826                 bool huge)
 827 {
 828         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 829                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 830         unsigned int cur_page, n_pages, pgsz_idx;
 831         size_t mem_sz, cur_pgsz;
 832         rte_iova_t *iovas = NULL;
 833         void *addr;
 834         int ret;
 835
 836         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 837                 /* skip anything that is too big */
 838                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 839                         continue;
 840
 841                 cur_pgsz = pgsizes[pgsz_idx];
 842
 843                 /* if we were told not to allocate hugepages, override */
 844                 if (!huge)
 845                         cur_pgsz = sysconf(_SC_PAGESIZE);
 846
 847                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 848                 if (ret < 0) {
 849                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 850                         return -1;
 851                 }
 852
 853                 /* allocate our memory */
 854                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 855
 856                 /* if we couldn't allocate memory with a specified page size,
 857                  * that doesn't mean we can't do it with other page sizes, so
 858                  * try another one.
 859                  */
 860                 if (addr == NULL)
 861                         continue;
 862
 863                 /* store IOVA addresses for every page in this memory area */
 864                 n_pages = mem_sz / cur_pgsz;
 865
 866                 iovas = malloc(sizeof(*iovas) * n_pages);
 867
 868                 if (iovas == NULL) {
 869                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 870                         goto fail;
 871                 }
 872                 /* lock memory if it's not huge pages */
 873                 if (!huge)
 874                         mlock(addr, mem_sz);
 875
 876                 /* populate IOVA addresses */
 877                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 878                         rte_iova_t iova;
 879                         size_t offset;
 880                         void *cur;
 881
 882                         offset = cur_pgsz * cur_page;
 883                         cur = RTE_PTR_ADD(addr, offset);
 884
 885                         /* touch the page before getting its IOVA */
 886                         *(volatile char *)cur = 0;
 887
 888                         iova = rte_mem_virt2iova(cur);
 889
 890                         iovas[cur_page] = iova;
 891                 }
 892
 893                 break;
 894         }
 895         /* if we couldn't allocate anything */
 896         if (iovas == NULL)
 897                 return -1;
 898
 899         param->addr = addr;
 900         param->len = mem_sz;
 901         param->pgsz = cur_pgsz;
 902         param->iova_table = iovas;
 903         param->iova_table_len = n_pages;
 904
 905         return 0;
 906 fail:
 907         if (iovas)
 908                 free(iovas);
 909         if (addr)
 910                 munmap(addr, mem_sz);
 911
 912         return -1;
 913 }
 914
 915 static int
 916 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 917 {
 918         struct extmem_param param;
 919         int socket_id, ret;
 920
 921         memset(&param, 0, sizeof(param));
 922
 923         /* check if our heap exists */
 924         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 925         if (socket_id < 0) {
 926                 /* create our heap */
 927                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 928                 if (ret < 0) {
 929                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 930                         return -1;
 931                 }
 932         }
 933
 934         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 935         if (ret < 0) {
 936                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 937                 return -1;
 938         }
 939
 940         /* we now have a valid memory area, so add it to heap */
 941         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 942                         param.addr, param.len, param.iova_table,
 943                         param.iova_table_len, param.pgsz);
 944
 945         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 946
 947         /* not needed any more */
 948         free(param.iova_table);
 949
 950         if (ret < 0) {
 951                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 952                 munmap(param.addr, param.len);
 953                 return -1;
 954         }
 955
 956         /* success */
 957
 958         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 959                         param.len >> 20);
 960
 961         return 0;
 962 }
 963 static void
 964 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 965              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 966 {
 967         uint16_t pid = 0;
 968         int ret;
 969
 970         RTE_ETH_FOREACH_DEV(pid) {
 971                 struct rte_eth_dev_info dev_info;
 972
 973                 ret = eth_dev_info_get_print_err(pid, &dev_info);
 974                 if (ret != 0) {
 975                         TESTPMD_LOG(DEBUG,
 976                                     "unable to get device info for port %d on addr 0x%p,"
 977                                     "mempool unmapping will not be performed\n",
 978                                     pid, memhdr->addr);
 979                         continue;
 980                 }
 981
 982                 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
 983                 if (ret) {
 984                         TESTPMD_LOG(DEBUG,
 985                                     "unable to DMA unmap addr 0x%p "
 986                                     "for device %s\n",
 987                                     memhdr->addr, dev_info.device->name);
 988                 }
 989         }
 990         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 991         if (ret) {
 992                 TESTPMD_LOG(DEBUG,
 993                             "unable to un-register addr 0x%p\n", memhdr->addr);
 994         }
 995 }
 996
 997 static void
 998 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 999            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1000 {
1001         uint16_t pid = 0;
1002         size_t page_size = sysconf(_SC_PAGESIZE);
1003         int ret;
1004
1005         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
1006                                   page_size);
1007         if (ret) {
1008                 TESTPMD_LOG(DEBUG,
1009                             "unable to register addr 0x%p\n", memhdr->addr);
1010                 return;
1011         }
1012         RTE_ETH_FOREACH_DEV(pid) {
1013                 struct rte_eth_dev_info dev_info;
1014
1015                 ret = eth_dev_info_get_print_err(pid, &dev_info);
1016                 if (ret != 0) {
1017                         TESTPMD_LOG(DEBUG,
1018                                     "unable to get device info for port %d on addr 0x%p,"
1019                                     "mempool mapping will not be performed\n",
1020                                     pid, memhdr->addr);
1021                         continue;
1022                 }
1023                 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
1024                 if (ret) {
1025                         TESTPMD_LOG(DEBUG,
1026                                     "unable to DMA map addr 0x%p "
1027                                     "for device %s\n",
1028                                     memhdr->addr, dev_info.device->name);
1029                 }
1030         }
1031 }
1032 #endif
1033
1034 static unsigned int
1035 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
1036             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
1037 {
1038         struct rte_pktmbuf_extmem *xmem;
1039         unsigned int ext_num, zone_num, elt_num;
1040         uint16_t elt_size;
1041
1042         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
1043         elt_num = EXTBUF_ZONE_SIZE / elt_size;
1044         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
1045
1046         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
1047         if (xmem == NULL) {
1048                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
1049                                  "external buffer descriptors\n");
1050                 *ext_mem = NULL;
1051                 return 0;
1052         }
1053         for (ext_num = 0; ext_num < zone_num; ext_num++) {
1054                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
1055                 const struct rte_memzone *mz;
1056                 char mz_name[RTE_MEMZONE_NAMESIZE];
1057                 int ret;
1058
1059                 ret = snprintf(mz_name, sizeof(mz_name),
1060                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
1061                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
1062                         errno = ENAMETOOLONG;
1063                         ext_num = 0;
1064                         break;
1065                 }
1066                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
1067                                                  socket_id,
1068                                                  RTE_MEMZONE_IOVA_CONTIG |
1069                                                  RTE_MEMZONE_1GB |
1070                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
1071                                                  EXTBUF_ZONE_SIZE);
1072                 if (mz == NULL) {
1073                         /*
1074                          * The caller exits on external buffer creation
1075                          * error, so there is no need to free memzones.
1076                          */
1077                         errno = ENOMEM;
1078                         ext_num = 0;
1079                         break;
1080                 }
1081                 xseg->buf_ptr = mz->addr;
1082                 xseg->buf_iova = mz->iova;
1083                 xseg->buf_len = EXTBUF_ZONE_SIZE;
1084                 xseg->elt_size = elt_size;
1085         }
1086         if (ext_num == 0 && xmem != NULL) {
1087                 free(xmem);
1088                 xmem = NULL;
1089         }
1090         *ext_mem = xmem;
1091         return ext_num;
1092 }
1093
1094 /*
1095  * Configuration initialisation done once at init time.
1096  */
1097 static struct rte_mempool *
1098 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
1099                  unsigned int socket_id, uint16_t size_idx)
1100 {
1101         char pool_name[RTE_MEMPOOL_NAMESIZE];
1102         struct rte_mempool *rte_mp = NULL;
1103 #ifndef RTE_EXEC_ENV_WINDOWS
1104         uint32_t mb_size;
1105
1106         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
1107 #endif
1108         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
1109         if (!is_proc_primary()) {
1110                 rte_mp = rte_mempool_lookup(pool_name);
1111                 if (rte_mp == NULL)
1112                         rte_exit(EXIT_FAILURE,
1113                                 "Get mbuf pool for socket %u failed: %s\n",
1114                                 socket_id, rte_strerror(rte_errno));
1115                 return rte_mp;
1116         }
1117
1118         TESTPMD_LOG(INFO,
1119                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1120                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1121
1122         switch (mp_alloc_type) {
1123         case MP_ALLOC_NATIVE:
1124                 {
1125                         /* wrapper to rte_mempool_create() */
1126                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1127                                         rte_mbuf_best_mempool_ops());
1128                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1129                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1130                         break;
1131                 }
1132 #ifndef RTE_EXEC_ENV_WINDOWS
1133         case MP_ALLOC_ANON:
1134                 {
1135                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1136                                 mb_size, (unsigned int) mb_mempool_cache,
1137                                 sizeof(struct rte_pktmbuf_pool_private),
1138                                 socket_id, mempool_flags);
1139                         if (rte_mp == NULL)
1140                                 goto err;
1141
1142                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1143                                 rte_mempool_free(rte_mp);
1144                                 rte_mp = NULL;
1145                                 goto err;
1146                         }
1147                         rte_pktmbuf_pool_init(rte_mp, NULL);
1148                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1149                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1150                         break;
1151                 }
1152         case MP_ALLOC_XMEM:
1153         case MP_ALLOC_XMEM_HUGE:
1154                 {
1155                         int heap_socket;
1156                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1157
1158                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1159                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1160
1161                         heap_socket =
1162                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1163                         if (heap_socket < 0)
1164                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1165
1166                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1167                                         rte_mbuf_best_mempool_ops());
1168                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1169                                         mb_mempool_cache, 0, mbuf_seg_size,
1170                                         heap_socket);
1171                         break;
1172                 }
1173 #endif
1174         case MP_ALLOC_XBUF:
1175                 {
1176                         struct rte_pktmbuf_extmem *ext_mem;
1177                         unsigned int ext_num;
1178
1179                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1180                                                socket_id, pool_name, &ext_mem);
1181                         if (ext_num == 0)
1182                                 rte_exit(EXIT_FAILURE,
1183                                          "Can't create pinned data buffers\n");
1184
1185                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1186                                         rte_mbuf_best_mempool_ops());
1187                         rte_mp = rte_pktmbuf_pool_create_extbuf
1188                                         (pool_name, nb_mbuf, mb_mempool_cache,
1189                                          0, mbuf_seg_size, socket_id,
1190                                          ext_mem, ext_num);
1191                         free(ext_mem);
1192                         break;
1193                 }
1194         default:
1195                 {
1196                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1197                 }
1198         }
1199
1200 #ifndef RTE_EXEC_ENV_WINDOWS
1201 err:
1202 #endif
1203         if (rte_mp == NULL) {
1204                 rte_exit(EXIT_FAILURE,
1205                         "Creation of mbuf pool for socket %u failed: %s\n",
1206                         socket_id, rte_strerror(rte_errno));
1207         } else if (verbose_level > 0) {
1208                 rte_mempool_dump(stdout, rte_mp);
1209         }
1210         return rte_mp;
1211 }
1212
1213 /*
1214  * Check given socket id is valid or not with NUMA mode,
1215  * if valid, return 0, else return -1
1216  */
1217 static int
1218 check_socket_id(const unsigned int socket_id)
1219 {
1220         static int warning_once = 0;
1221
1222         if (new_socket_id(socket_id)) {
1223                 if (!warning_once && numa_support)
1224                         fprintf(stderr,
1225                                 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
1226                 warning_once = 1;
1227                 return -1;
1228         }
1229         return 0;
1230 }
1231
1232 /*
1233  * Get the allowed maximum number of RX queues.
1234  * *pid return the port id which has minimal value of
1235  * max_rx_queues in all ports.
1236  */
1237 queueid_t
1238 get_allowed_max_nb_rxq(portid_t *pid)
1239 {
1240         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1241         bool max_rxq_valid = false;
1242         portid_t pi;
1243         struct rte_eth_dev_info dev_info;
1244
1245         RTE_ETH_FOREACH_DEV(pi) {
1246                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1247                         continue;
1248
1249                 max_rxq_valid = true;
1250                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1251                         allowed_max_rxq = dev_info.max_rx_queues;
1252                         *pid = pi;
1253                 }
1254         }
1255         return max_rxq_valid ? allowed_max_rxq : 0;
1256 }
1257
1258 /*
1259  * Check input rxq is valid or not.
1260  * If input rxq is not greater than any of maximum number
1261  * of RX queues of all ports, it is valid.
1262  * if valid, return 0, else return -1
1263  */
1264 int
1265 check_nb_rxq(queueid_t rxq)
1266 {
1267         queueid_t allowed_max_rxq;
1268         portid_t pid = 0;
1269
1270         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1271         if (rxq > allowed_max_rxq) {
1272                 fprintf(stderr,
1273                         "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
1274                         rxq, allowed_max_rxq, pid);
1275                 return -1;
1276         }
1277         return 0;
1278 }
1279
1280 /*
1281  * Get the allowed maximum number of TX queues.
1282  * *pid return the port id which has minimal value of
1283  * max_tx_queues in all ports.
1284  */
1285 queueid_t
1286 get_allowed_max_nb_txq(portid_t *pid)
1287 {
1288         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1289         bool max_txq_valid = false;
1290         portid_t pi;
1291         struct rte_eth_dev_info dev_info;
1292
1293         RTE_ETH_FOREACH_DEV(pi) {
1294                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1295                         continue;
1296
1297                 max_txq_valid = true;
1298                 if (dev_info.max_tx_queues < allowed_max_txq) {
1299                         allowed_max_txq = dev_info.max_tx_queues;
1300                         *pid = pi;
1301                 }
1302         }
1303         return max_txq_valid ? allowed_max_txq : 0;
1304 }
1305
1306 /*
1307  * Check input txq is valid or not.
1308  * If input txq is not greater than any of maximum number
1309  * of TX queues of all ports, it is valid.
1310  * if valid, return 0, else return -1
1311  */
1312 int
1313 check_nb_txq(queueid_t txq)
1314 {
1315         queueid_t allowed_max_txq;
1316         portid_t pid = 0;
1317
1318         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1319         if (txq > allowed_max_txq) {
1320                 fprintf(stderr,
1321                         "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n",
1322                         txq, allowed_max_txq, pid);
1323                 return -1;
1324         }
1325         return 0;
1326 }
1327
1328 /*
1329  * Get the allowed maximum number of RXDs of every rx queue.
1330  * *pid return the port id which has minimal value of
1331  * max_rxd in all queues of all ports.
1332  */
1333 static uint16_t
1334 get_allowed_max_nb_rxd(portid_t *pid)
1335 {
1336         uint16_t allowed_max_rxd = UINT16_MAX;
1337         portid_t pi;
1338         struct rte_eth_dev_info dev_info;
1339
1340         RTE_ETH_FOREACH_DEV(pi) {
1341                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1342                         continue;
1343
1344                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1345                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1346                         *pid = pi;
1347                 }
1348         }
1349         return allowed_max_rxd;
1350 }
1351
1352 /*
1353  * Get the allowed minimal number of RXDs of every rx queue.
1354  * *pid return the port id which has minimal value of
1355  * min_rxd in all queues of all ports.
1356  */
1357 static uint16_t
1358 get_allowed_min_nb_rxd(portid_t *pid)
1359 {
1360         uint16_t allowed_min_rxd = 0;
1361         portid_t pi;
1362         struct rte_eth_dev_info dev_info;
1363
1364         RTE_ETH_FOREACH_DEV(pi) {
1365                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1366                         continue;
1367
1368                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1369                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1370                         *pid = pi;
1371                 }
1372         }
1373
1374         return allowed_min_rxd;
1375 }
1376
1377 /*
1378  * Check input rxd is valid or not.
1379  * If input rxd is not greater than any of maximum number
1380  * of RXDs of every Rx queues and is not less than any of
1381  * minimal number of RXDs of every Rx queues, it is valid.
1382  * if valid, return 0, else return -1
1383  */
1384 int
1385 check_nb_rxd(queueid_t rxd)
1386 {
1387         uint16_t allowed_max_rxd;
1388         uint16_t allowed_min_rxd;
1389         portid_t pid = 0;
1390
1391         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1392         if (rxd > allowed_max_rxd) {
1393                 fprintf(stderr,
1394                         "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n",
1395                         rxd, allowed_max_rxd, pid);
1396                 return -1;
1397         }
1398
1399         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1400         if (rxd < allowed_min_rxd) {
1401                 fprintf(stderr,
1402                         "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n",
1403                         rxd, allowed_min_rxd, pid);
1404                 return -1;
1405         }
1406
1407         return 0;
1408 }
1409
1410 /*
1411  * Get the allowed maximum number of TXDs of every rx queues.
1412  * *pid return the port id which has minimal value of
1413  * max_txd in every tx queue.
1414  */
1415 static uint16_t
1416 get_allowed_max_nb_txd(portid_t *pid)
1417 {
1418         uint16_t allowed_max_txd = UINT16_MAX;
1419         portid_t pi;
1420         struct rte_eth_dev_info dev_info;
1421
1422         RTE_ETH_FOREACH_DEV(pi) {
1423                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1424                         continue;
1425
1426                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1427                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1428                         *pid = pi;
1429                 }
1430         }
1431         return allowed_max_txd;
1432 }
1433
1434 /*
1435  * Get the allowed maximum number of TXDs of every tx queues.
1436  * *pid return the port id which has minimal value of
1437  * min_txd in every tx queue.
1438  */
1439 static uint16_t
1440 get_allowed_min_nb_txd(portid_t *pid)
1441 {
1442         uint16_t allowed_min_txd = 0;
1443         portid_t pi;
1444         struct rte_eth_dev_info dev_info;
1445
1446         RTE_ETH_FOREACH_DEV(pi) {
1447                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1448                         continue;
1449
1450                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1451                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1452                         *pid = pi;
1453                 }
1454         }
1455
1456         return allowed_min_txd;
1457 }
1458
1459 /*
1460  * Check input txd is valid or not.
1461  * If input txd is not greater than any of maximum number
1462  * of TXDs of every Rx queues, it is valid.
1463  * if valid, return 0, else return -1
1464  */
1465 int
1466 check_nb_txd(queueid_t txd)
1467 {
1468         uint16_t allowed_max_txd;
1469         uint16_t allowed_min_txd;
1470         portid_t pid = 0;
1471
1472         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1473         if (txd > allowed_max_txd) {
1474                 fprintf(stderr,
1475                         "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n",
1476                         txd, allowed_max_txd, pid);
1477                 return -1;
1478         }
1479
1480         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1481         if (txd < allowed_min_txd) {
1482                 fprintf(stderr,
1483                         "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n",
1484                         txd, allowed_min_txd, pid);
1485                 return -1;
1486         }
1487         return 0;
1488 }
1489
1490
1491 /*
1492  * Get the allowed maximum number of hairpin queues.
1493  * *pid return the port id which has minimal value of
1494  * max_hairpin_queues in all ports.
1495  */
1496 queueid_t
1497 get_allowed_max_nb_hairpinq(portid_t *pid)
1498 {
1499         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1500         portid_t pi;
1501         struct rte_eth_hairpin_cap cap;
1502
1503         RTE_ETH_FOREACH_DEV(pi) {
1504                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1505                         *pid = pi;
1506                         return 0;
1507                 }
1508                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1509                         allowed_max_hairpinq = cap.max_nb_queues;
1510                         *pid = pi;
1511                 }
1512         }
1513         return allowed_max_hairpinq;
1514 }
1515
1516 /*
1517  * Check input hairpin is valid or not.
1518  * If input hairpin is not greater than any of maximum number
1519  * of hairpin queues of all ports, it is valid.
1520  * if valid, return 0, else return -1
1521  */
1522 int
1523 check_nb_hairpinq(queueid_t hairpinq)
1524 {
1525         queueid_t allowed_max_hairpinq;
1526         portid_t pid = 0;
1527
1528         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1529         if (hairpinq > allowed_max_hairpinq) {
1530                 fprintf(stderr,
1531                         "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n",
1532                         hairpinq, allowed_max_hairpinq, pid);
1533                 return -1;
1534         }
1535         return 0;
1536 }
1537
1538 static void
1539 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1540 {
1541         struct rte_port *port = &ports[pid];
1542         uint16_t data_size;
1543         int ret;
1544         int i;
1545
1546         eth_rx_metadata_negotiate_mp(pid);
1547         flow_pick_transfer_proxy_mp(pid);
1548
1549         port->dev_conf.txmode = tx_mode;
1550         port->dev_conf.rxmode = rx_mode;
1551
1552         ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1553         if (ret != 0)
1554                 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1555
1556         ret = update_jumbo_frame_offload(pid);
1557         if (ret != 0)
1558                 fprintf(stderr,
1559                         "Updating jumbo frame offload failed for port %u\n",
1560                         pid);
1561
1562         if (!(port->dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1563                 port->dev_conf.txmode.offloads &=
1564                         ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1565
1566         /* Apply Rx offloads configuration */
1567         for (i = 0; i < port->dev_info.max_rx_queues; i++)
1568                 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads;
1569         /* Apply Tx offloads configuration */
1570         for (i = 0; i < port->dev_info.max_tx_queues; i++)
1571                 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads;
1572
1573         if (eth_link_speed)
1574                 port->dev_conf.link_speeds = eth_link_speed;
1575
1576         /* set flag to initialize port/queue */
1577         port->need_reconfig = 1;
1578         port->need_reconfig_queues = 1;
1579         port->socket_id = socket_id;
1580         port->tx_metadata = 0;
1581
1582         /*
1583          * Check for maximum number of segments per MTU.
1584          * Accordingly update the mbuf data size.
1585          */
1586         if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1587             port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1588                 data_size = rx_mode.max_rx_pkt_len /
1589                         port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1590
1591                 if ((data_size + RTE_PKTMBUF_HEADROOM) > mbuf_data_size[0]) {
1592                         mbuf_data_size[0] = data_size + RTE_PKTMBUF_HEADROOM;
1593                         TESTPMD_LOG(WARNING,
1594                                     "Configured mbuf size of the first segment %hu\n",
1595                                     mbuf_data_size[0]);
1596                 }
1597         }
1598 }
1599
1600 static void
1601 init_config(void)
1602 {
1603         portid_t pid;
1604         struct rte_mempool *mbp;
1605         unsigned int nb_mbuf_per_pool;
1606         lcoreid_t  lc_id;
1607         struct rte_gro_param gro_param;
1608         uint32_t gso_types;
1609
1610         /* Configuration of logical cores. */
1611         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1612                                 sizeof(struct fwd_lcore *) * nb_lcores,
1613                                 RTE_CACHE_LINE_SIZE);
1614         if (fwd_lcores == NULL) {
1615                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1616                                                         "failed\n", nb_lcores);
1617         }
1618         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1619                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1620                                                sizeof(struct fwd_lcore),
1621                                                RTE_CACHE_LINE_SIZE);
1622                 if (fwd_lcores[lc_id] == NULL) {
1623                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1624                                                                 "failed\n");
1625                 }
1626                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1627         }
1628
1629         RTE_ETH_FOREACH_DEV(pid) {
1630                 uint32_t socket_id;
1631
1632                 if (numa_support) {
1633                         socket_id = port_numa[pid];
1634                         if (port_numa[pid] == NUMA_NO_CONFIG) {
1635                                 socket_id = rte_eth_dev_socket_id(pid);
1636
1637                                 /*
1638                                  * if socket_id is invalid,
1639                                  * set to the first available socket.
1640                                  */
1641                                 if (check_socket_id(socket_id) < 0)
1642                                         socket_id = socket_ids[0];
1643                         }
1644                 } else {
1645                         socket_id = (socket_num == UMA_NO_CONFIG) ?
1646                                     0 : socket_num;
1647                 }
1648                 /* Apply default TxRx configuration for all ports */
1649                 init_config_port_offloads(pid, socket_id);
1650         }
1651         /*
1652          * Create pools of mbuf.
1653          * If NUMA support is disabled, create a single pool of mbuf in
1654          * socket 0 memory by default.
1655          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1656          *
1657          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1658          * nb_txd can be configured at run time.
1659          */
1660         if (param_total_num_mbufs)
1661                 nb_mbuf_per_pool = param_total_num_mbufs;
1662         else {
1663                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1664                         (nb_lcores * mb_mempool_cache) +
1665                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1666                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1667         }
1668
1669         if (numa_support) {
1670                 uint8_t i, j;
1671
1672                 for (i = 0; i < num_sockets; i++)
1673                         for (j = 0; j < mbuf_data_size_n; j++)
1674                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1675                                         mbuf_pool_create(mbuf_data_size[j],
1676                                                           nb_mbuf_per_pool,
1677                                                           socket_ids[i], j);
1678         } else {
1679                 uint8_t i;
1680
1681                 for (i = 0; i < mbuf_data_size_n; i++)
1682                         mempools[i] = mbuf_pool_create
1683                                         (mbuf_data_size[i],
1684                                          nb_mbuf_per_pool,
1685                                          socket_num == UMA_NO_CONFIG ?
1686                                          0 : socket_num, i);
1687         }
1688
1689         init_port_config();
1690
1691         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1692                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1693         /*
1694          * Records which Mbuf pool to use by each logical core, if needed.
1695          */
1696         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1697                 mbp = mbuf_pool_find(
1698                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1699
1700                 if (mbp == NULL)
1701                         mbp = mbuf_pool_find(0, 0);
1702                 fwd_lcores[lc_id]->mbp = mbp;
1703                 /* initialize GSO context */
1704                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1705                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1706                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1707                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1708                         RTE_ETHER_CRC_LEN;
1709                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1710         }
1711
1712         fwd_config_setup();
1713
1714         /* create a gro context for each lcore */
1715         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1716         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1717         gro_param.max_item_per_flow = MAX_PKT_BURST;
1718         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1719                 gro_param.socket_id = rte_lcore_to_socket_id(
1720                                 fwd_lcores_cpuids[lc_id]);
1721                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1722                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1723                         rte_exit(EXIT_FAILURE,
1724                                         "rte_gro_ctx_create() failed\n");
1725                 }
1726         }
1727 }
1728
1729
1730 void
1731 reconfig(portid_t new_port_id, unsigned socket_id)
1732 {
1733         /* Reconfiguration of Ethernet ports. */
1734         init_config_port_offloads(new_port_id, socket_id);
1735         init_port_config();
1736 }
1737
1738
1739 int
1740 init_fwd_streams(void)
1741 {
1742         portid_t pid;
1743         struct rte_port *port;
1744         streamid_t sm_id, nb_fwd_streams_new;
1745         queueid_t q;
1746
1747         /* set socket id according to numa or not */
1748         RTE_ETH_FOREACH_DEV(pid) {
1749                 port = &ports[pid];
1750                 if (nb_rxq > port->dev_info.max_rx_queues) {
1751                         fprintf(stderr,
1752                                 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n",
1753                                 nb_rxq, port->dev_info.max_rx_queues);
1754                         return -1;
1755                 }
1756                 if (nb_txq > port->dev_info.max_tx_queues) {
1757                         fprintf(stderr,
1758                                 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n",
1759                                 nb_txq, port->dev_info.max_tx_queues);
1760                         return -1;
1761                 }
1762                 if (numa_support) {
1763                         if (port_numa[pid] != NUMA_NO_CONFIG)
1764                                 port->socket_id = port_numa[pid];
1765                         else {
1766                                 port->socket_id = rte_eth_dev_socket_id(pid);
1767
1768                                 /*
1769                                  * if socket_id is invalid,
1770                                  * set to the first available socket.
1771                                  */
1772                                 if (check_socket_id(port->socket_id) < 0)
1773                                         port->socket_id = socket_ids[0];
1774                         }
1775                 }
1776                 else {
1777                         if (socket_num == UMA_NO_CONFIG)
1778                                 port->socket_id = 0;
1779                         else
1780                                 port->socket_id = socket_num;
1781                 }
1782         }
1783
1784         q = RTE_MAX(nb_rxq, nb_txq);
1785         if (q == 0) {
1786                 fprintf(stderr,
1787                         "Fail: Cannot allocate fwd streams as number of queues is 0\n");
1788                 return -1;
1789         }
1790         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1791         if (nb_fwd_streams_new == nb_fwd_streams)
1792                 return 0;
1793         /* clear the old */
1794         if (fwd_streams != NULL) {
1795                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1796                         if (fwd_streams[sm_id] == NULL)
1797                                 continue;
1798                         rte_free(fwd_streams[sm_id]);
1799                         fwd_streams[sm_id] = NULL;
1800                 }
1801                 rte_free(fwd_streams);
1802                 fwd_streams = NULL;
1803         }
1804
1805         /* init new */
1806         nb_fwd_streams = nb_fwd_streams_new;
1807         if (nb_fwd_streams) {
1808                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1809                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1810                         RTE_CACHE_LINE_SIZE);
1811                 if (fwd_streams == NULL)
1812                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1813                                  " (struct fwd_stream *)) failed\n",
1814                                  nb_fwd_streams);
1815
1816                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1817                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1818                                 " struct fwd_stream", sizeof(struct fwd_stream),
1819                                 RTE_CACHE_LINE_SIZE);
1820                         if (fwd_streams[sm_id] == NULL)
1821                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1822                                          "(struct fwd_stream) failed\n");
1823                 }
1824         }
1825
1826         return 0;
1827 }
1828
1829 static void
1830 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1831 {
1832         uint64_t total_burst, sburst;
1833         uint64_t nb_burst;
1834         uint64_t burst_stats[4];
1835         uint16_t pktnb_stats[4];
1836         uint16_t nb_pkt;
1837         int burst_percent[4], sburstp;
1838         int i;
1839
1840         /*
1841          * First compute the total number of packet bursts and the
1842          * two highest numbers of bursts of the same number of packets.
1843          */
1844         memset(&burst_stats, 0x0, sizeof(burst_stats));
1845         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1846
1847         /* Show stats for 0 burst size always */
1848         total_burst = pbs->pkt_burst_spread[0];
1849         burst_stats[0] = pbs->pkt_burst_spread[0];
1850         pktnb_stats[0] = 0;
1851
1852         /* Find the next 2 burst sizes with highest occurrences. */
1853         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1854                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1855
1856                 if (nb_burst == 0)
1857                         continue;
1858
1859                 total_burst += nb_burst;
1860
1861                 if (nb_burst > burst_stats[1]) {
1862                         burst_stats[2] = burst_stats[1];
1863                         pktnb_stats[2] = pktnb_stats[1];
1864                         burst_stats[1] = nb_burst;
1865                         pktnb_stats[1] = nb_pkt;
1866                 } else if (nb_burst > burst_stats[2]) {
1867                         burst_stats[2] = nb_burst;
1868                         pktnb_stats[2] = nb_pkt;
1869                 }
1870         }
1871         if (total_burst == 0)
1872                 return;
1873
1874         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1875         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1876                 if (i == 3) {
1877                         printf("%d%% of other]\n", 100 - sburstp);
1878                         return;
1879                 }
1880
1881                 sburst += burst_stats[i];
1882                 if (sburst == total_burst) {
1883                         printf("%d%% of %d pkts]\n",
1884                                 100 - sburstp, (int) pktnb_stats[i]);
1885                         return;
1886                 }
1887
1888                 burst_percent[i] =
1889                         (double)burst_stats[i] / total_burst * 100;
1890                 printf("%d%% of %d pkts + ",
1891                         burst_percent[i], (int) pktnb_stats[i]);
1892                 sburstp += burst_percent[i];
1893         }
1894 }
1895
1896 static void
1897 fwd_stream_stats_display(streamid_t stream_id)
1898 {
1899         struct fwd_stream *fs;
1900         static const char *fwd_top_stats_border = "-------";
1901
1902         fs = fwd_streams[stream_id];
1903         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1904             (fs->fwd_dropped == 0))
1905                 return;
1906         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1907                "TX Port=%2d/Queue=%2d %s\n",
1908                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1909                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1910         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1911                " TX-dropped: %-14"PRIu64,
1912                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1913
1914         /* if checksum mode */
1915         if (cur_fwd_eng == &csum_fwd_engine) {
1916                 printf("  RX- bad IP checksum: %-14"PRIu64
1917                        "  Rx- bad L4 checksum: %-14"PRIu64
1918                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1919                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1920                         fs->rx_bad_outer_l4_csum);
1921                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1922                         fs->rx_bad_outer_ip_csum);
1923         } else {
1924                 printf("\n");
1925         }
1926
1927         if (record_burst_stats) {
1928                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1929                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1930         }
1931 }
1932
1933 void
1934 fwd_stats_display(void)
1935 {
1936         static const char *fwd_stats_border = "----------------------";
1937         static const char *acc_stats_border = "+++++++++++++++";
1938         struct {
1939                 struct fwd_stream *rx_stream;
1940                 struct fwd_stream *tx_stream;
1941                 uint64_t tx_dropped;
1942                 uint64_t rx_bad_ip_csum;
1943                 uint64_t rx_bad_l4_csum;
1944                 uint64_t rx_bad_outer_l4_csum;
1945                 uint64_t rx_bad_outer_ip_csum;
1946         } ports_stats[RTE_MAX_ETHPORTS];
1947         uint64_t total_rx_dropped = 0;
1948         uint64_t total_tx_dropped = 0;
1949         uint64_t total_rx_nombuf = 0;
1950         struct rte_eth_stats stats;
1951         uint64_t fwd_cycles = 0;
1952         uint64_t total_recv = 0;
1953         uint64_t total_xmit = 0;
1954         struct rte_port *port;
1955         streamid_t sm_id;
1956         portid_t pt_id;
1957         int i;
1958
1959         memset(ports_stats, 0, sizeof(ports_stats));
1960
1961         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1962                 struct fwd_stream *fs = fwd_streams[sm_id];
1963
1964                 if (cur_fwd_config.nb_fwd_streams >
1965                     cur_fwd_config.nb_fwd_ports) {
1966                         fwd_stream_stats_display(sm_id);
1967                 } else {
1968                         ports_stats[fs->tx_port].tx_stream = fs;
1969                         ports_stats[fs->rx_port].rx_stream = fs;
1970                 }
1971
1972                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1973
1974                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1975                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1976                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1977                                 fs->rx_bad_outer_l4_csum;
1978                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
1979                                 fs->rx_bad_outer_ip_csum;
1980
1981                 if (record_core_cycles)
1982                         fwd_cycles += fs->core_cycles;
1983         }
1984         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1985                 pt_id = fwd_ports_ids[i];
1986                 port = &ports[pt_id];
1987
1988                 rte_eth_stats_get(pt_id, &stats);
1989                 stats.ipackets -= port->stats.ipackets;
1990                 stats.opackets -= port->stats.opackets;
1991                 stats.ibytes -= port->stats.ibytes;
1992                 stats.obytes -= port->stats.obytes;
1993                 stats.imissed -= port->stats.imissed;
1994                 stats.oerrors -= port->stats.oerrors;
1995                 stats.rx_nombuf -= port->stats.rx_nombuf;
1996
1997                 total_recv += stats.ipackets;
1998                 total_xmit += stats.opackets;
1999                 total_rx_dropped += stats.imissed;
2000                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
2001                 total_tx_dropped += stats.oerrors;
2002                 total_rx_nombuf  += stats.rx_nombuf;
2003
2004                 printf("\n  %s Forward statistics for port %-2d %s\n",
2005                        fwd_stats_border, pt_id, fwd_stats_border);
2006
2007                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
2008                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
2009                        stats.ipackets + stats.imissed);
2010
2011                 if (cur_fwd_eng == &csum_fwd_engine) {
2012                         printf("  Bad-ipcsum: %-14"PRIu64
2013                                " Bad-l4csum: %-14"PRIu64
2014                                "Bad-outer-l4csum: %-14"PRIu64"\n",
2015                                ports_stats[pt_id].rx_bad_ip_csum,
2016                                ports_stats[pt_id].rx_bad_l4_csum,
2017                                ports_stats[pt_id].rx_bad_outer_l4_csum);
2018                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
2019                                ports_stats[pt_id].rx_bad_outer_ip_csum);
2020                 }
2021                 if (stats.ierrors + stats.rx_nombuf > 0) {
2022                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
2023                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
2024                 }
2025
2026                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
2027                        "TX-total: %-"PRIu64"\n",
2028                        stats.opackets, ports_stats[pt_id].tx_dropped,
2029                        stats.opackets + ports_stats[pt_id].tx_dropped);
2030
2031                 if (record_burst_stats) {
2032                         if (ports_stats[pt_id].rx_stream)
2033                                 pkt_burst_stats_display("RX",
2034                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
2035                         if (ports_stats[pt_id].tx_stream)
2036                                 pkt_burst_stats_display("TX",
2037                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
2038                 }
2039
2040                 printf("  %s--------------------------------%s\n",
2041                        fwd_stats_border, fwd_stats_border);
2042         }
2043
2044         printf("\n  %s Accumulated forward statistics for all ports"
2045                "%s\n",
2046                acc_stats_border, acc_stats_border);
2047         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
2048                "%-"PRIu64"\n"
2049                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
2050                "%-"PRIu64"\n",
2051                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
2052                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
2053         if (total_rx_nombuf > 0)
2054                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
2055         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
2056                "%s\n",
2057                acc_stats_border, acc_stats_border);
2058         if (record_core_cycles) {
2059 #define CYC_PER_MHZ 1E6
2060                 if (total_recv > 0 || total_xmit > 0) {
2061                         uint64_t total_pkts = 0;
2062                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
2063                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
2064                                 total_pkts = total_xmit;
2065                         else
2066                                 total_pkts = total_recv;
2067
2068                         printf("\n  CPU cycles/packet=%.2F (total cycles="
2069                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
2070                                " MHz Clock\n",
2071                                (double) fwd_cycles / total_pkts,
2072                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2073                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2074                 }
2075         }
2076 }
2077
2078 void
2079 fwd_stats_reset(void)
2080 {
2081         streamid_t sm_id;
2082         portid_t pt_id;
2083         int i;
2084
2085         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2086                 pt_id = fwd_ports_ids[i];
2087                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2088         }
2089         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2090                 struct fwd_stream *fs = fwd_streams[sm_id];
2091
2092                 fs->rx_packets = 0;
2093                 fs->tx_packets = 0;
2094                 fs->fwd_dropped = 0;
2095                 fs->rx_bad_ip_csum = 0;
2096                 fs->rx_bad_l4_csum = 0;
2097                 fs->rx_bad_outer_l4_csum = 0;
2098                 fs->rx_bad_outer_ip_csum = 0;
2099
2100                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2101                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2102                 fs->core_cycles = 0;
2103         }
2104 }
2105
2106 static void
2107 flush_fwd_rx_queues(void)
2108 {
2109         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2110         portid_t  rxp;
2111         portid_t port_id;
2112         queueid_t rxq;
2113         uint16_t  nb_rx;
2114         uint16_t  i;
2115         uint8_t   j;
2116         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2117         uint64_t timer_period;
2118
2119         if (num_procs > 1) {
2120                 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n");
2121                 return;
2122         }
2123
2124         /* convert to number of cycles */
2125         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2126
2127         for (j = 0; j < 2; j++) {
2128                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2129                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2130                                 port_id = fwd_ports_ids[rxp];
2131                                 /**
2132                                 * testpmd can stuck in the below do while loop
2133                                 * if rte_eth_rx_burst() always returns nonzero
2134                                 * packets. So timer is added to exit this loop
2135                                 * after 1sec timer expiry.
2136                                 */
2137                                 prev_tsc = rte_rdtsc();
2138                                 do {
2139                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2140                                                 pkts_burst, MAX_PKT_BURST);
2141                                         for (i = 0; i < nb_rx; i++)
2142                                                 rte_pktmbuf_free(pkts_burst[i]);
2143
2144                                         cur_tsc = rte_rdtsc();
2145                                         diff_tsc = cur_tsc - prev_tsc;
2146                                         timer_tsc += diff_tsc;
2147                                 } while ((nb_rx > 0) &&
2148                                         (timer_tsc < timer_period));
2149                                 timer_tsc = 0;
2150                         }
2151                 }
2152                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2153         }
2154 }
2155
2156 static void
2157 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2158 {
2159         struct fwd_stream **fsm;
2160         streamid_t nb_fs;
2161         streamid_t sm_id;
2162 #ifdef RTE_LIB_BITRATESTATS
2163         uint64_t tics_per_1sec;
2164         uint64_t tics_datum;
2165         uint64_t tics_current;
2166         uint16_t i, cnt_ports;
2167
2168         cnt_ports = nb_ports;
2169         tics_datum = rte_rdtsc();
2170         tics_per_1sec = rte_get_timer_hz();
2171 #endif
2172         fsm = &fwd_streams[fc->stream_idx];
2173         nb_fs = fc->stream_nb;
2174         do {
2175                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2176                         (*pkt_fwd)(fsm[sm_id]);
2177 #ifdef RTE_LIB_BITRATESTATS
2178                 if (bitrate_enabled != 0 &&
2179                                 bitrate_lcore_id == rte_lcore_id()) {
2180                         tics_current = rte_rdtsc();
2181                         if (tics_current - tics_datum >= tics_per_1sec) {
2182                                 /* Periodic bitrate calculation */
2183                                 for (i = 0; i < cnt_ports; i++)
2184                                         rte_stats_bitrate_calc(bitrate_data,
2185                                                 ports_ids[i]);
2186                                 tics_datum = tics_current;
2187                         }
2188                 }
2189 #endif
2190 #ifdef RTE_LIB_LATENCYSTATS
2191                 if (latencystats_enabled != 0 &&
2192                                 latencystats_lcore_id == rte_lcore_id())
2193                         rte_latencystats_update();
2194 #endif
2195
2196         } while (! fc->stopped);
2197 }
2198
2199 static int
2200 start_pkt_forward_on_core(void *fwd_arg)
2201 {
2202         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2203                              cur_fwd_config.fwd_eng->packet_fwd);
2204         return 0;
2205 }
2206
2207 /*
2208  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2209  * Used to start communication flows in network loopback test configurations.
2210  */
2211 static int
2212 run_one_txonly_burst_on_core(void *fwd_arg)
2213 {
2214         struct fwd_lcore *fwd_lc;
2215         struct fwd_lcore tmp_lcore;
2216
2217         fwd_lc = (struct fwd_lcore *) fwd_arg;
2218         tmp_lcore = *fwd_lc;
2219         tmp_lcore.stopped = 1;
2220         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2221         return 0;
2222 }
2223
2224 /*
2225  * Launch packet forwarding:
2226  *     - Setup per-port forwarding context.
2227  *     - launch logical cores with their forwarding configuration.
2228  */
2229 static void
2230 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2231 {
2232         unsigned int i;
2233         unsigned int lc_id;
2234         int diag;
2235
2236         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2237                 lc_id = fwd_lcores_cpuids[i];
2238                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2239                         fwd_lcores[i]->stopped = 0;
2240                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2241                                                      fwd_lcores[i], lc_id);
2242                         if (diag != 0)
2243                                 fprintf(stderr,
2244                                         "launch lcore %u failed - diag=%d\n",
2245                                         lc_id, diag);
2246                 }
2247         }
2248 }
2249
2250 /*
2251  * Launch packet forwarding configuration.
2252  */
2253 void
2254 start_packet_forwarding(int with_tx_first)
2255 {
2256         port_fwd_begin_t port_fwd_begin;
2257         port_fwd_end_t  port_fwd_end;
2258         unsigned int i;
2259
2260         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2261                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2262
2263         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2264                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2265
2266         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2267                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2268                 (!nb_rxq || !nb_txq))
2269                 rte_exit(EXIT_FAILURE,
2270                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2271                         cur_fwd_eng->fwd_mode_name);
2272
2273         if (all_ports_started() == 0) {
2274                 fprintf(stderr, "Not all ports were started\n");
2275                 return;
2276         }
2277         if (test_done == 0) {
2278                 fprintf(stderr, "Packet forwarding already started\n");
2279                 return;
2280         }
2281
2282         fwd_config_setup();
2283
2284         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2285         if (port_fwd_begin != NULL) {
2286                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2287                         if (port_fwd_begin(fwd_ports_ids[i])) {
2288                                 fprintf(stderr,
2289                                         "Packet forwarding is not ready\n");
2290                                 return;
2291                         }
2292                 }
2293         }
2294
2295         if (with_tx_first) {
2296                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2297                 if (port_fwd_begin != NULL) {
2298                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2299                                 if (port_fwd_begin(fwd_ports_ids[i])) {
2300                                         fprintf(stderr,
2301                                                 "Packet forwarding is not ready\n");
2302                                         return;
2303                                 }
2304                         }
2305                 }
2306         }
2307
2308         test_done = 0;
2309
2310         if(!no_flush_rx)
2311                 flush_fwd_rx_queues();
2312
2313         pkt_fwd_config_display(&cur_fwd_config);
2314         rxtx_config_display();
2315
2316         fwd_stats_reset();
2317         if (with_tx_first) {
2318                 while (with_tx_first--) {
2319                         launch_packet_forwarding(
2320                                         run_one_txonly_burst_on_core);
2321                         rte_eal_mp_wait_lcore();
2322                 }
2323                 port_fwd_end = tx_only_engine.port_fwd_end;
2324                 if (port_fwd_end != NULL) {
2325                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2326                                 (*port_fwd_end)(fwd_ports_ids[i]);
2327                 }
2328         }
2329         launch_packet_forwarding(start_pkt_forward_on_core);
2330 }
2331
2332 void
2333 stop_packet_forwarding(void)
2334 {
2335         port_fwd_end_t port_fwd_end;
2336         lcoreid_t lc_id;
2337         portid_t pt_id;
2338         int i;
2339
2340         if (test_done) {
2341                 fprintf(stderr, "Packet forwarding not started\n");
2342                 return;
2343         }
2344         printf("Telling cores to stop...");
2345         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2346                 fwd_lcores[lc_id]->stopped = 1;
2347         printf("\nWaiting for lcores to finish...\n");
2348         rte_eal_mp_wait_lcore();
2349         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2350         if (port_fwd_end != NULL) {
2351                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2352                         pt_id = fwd_ports_ids[i];
2353                         (*port_fwd_end)(pt_id);
2354                 }
2355         }
2356
2357         fwd_stats_display();
2358
2359         printf("\nDone.\n");
2360         test_done = 1;
2361 }
2362
2363 void
2364 dev_set_link_up(portid_t pid)
2365 {
2366         if (rte_eth_dev_set_link_up(pid) < 0)
2367                 fprintf(stderr, "\nSet link up fail.\n");
2368 }
2369
2370 void
2371 dev_set_link_down(portid_t pid)
2372 {
2373         if (rte_eth_dev_set_link_down(pid) < 0)
2374                 fprintf(stderr, "\nSet link down fail.\n");
2375 }
2376
2377 static int
2378 all_ports_started(void)
2379 {
2380         portid_t pi;
2381         struct rte_port *port;
2382
2383         RTE_ETH_FOREACH_DEV(pi) {
2384                 port = &ports[pi];
2385                 /* Check if there is a port which is not started */
2386                 if ((port->port_status != RTE_PORT_STARTED) &&
2387                         (port->slave_flag == 0))
2388                         return 0;
2389         }
2390
2391         /* No port is not started */
2392         return 1;
2393 }
2394
2395 int
2396 port_is_stopped(portid_t port_id)
2397 {
2398         struct rte_port *port = &ports[port_id];
2399
2400         if ((port->port_status != RTE_PORT_STOPPED) &&
2401             (port->slave_flag == 0))
2402                 return 0;
2403         return 1;
2404 }
2405
2406 int
2407 all_ports_stopped(void)
2408 {
2409         portid_t pi;
2410
2411         RTE_ETH_FOREACH_DEV(pi) {
2412                 if (!port_is_stopped(pi))
2413                         return 0;
2414         }
2415
2416         return 1;
2417 }
2418
2419 int
2420 port_is_started(portid_t port_id)
2421 {
2422         if (port_id_is_invalid(port_id, ENABLED_WARN))
2423                 return 0;
2424
2425         if (ports[port_id].port_status != RTE_PORT_STARTED)
2426                 return 0;
2427
2428         return 1;
2429 }
2430
2431 /* Configure the Rx and Tx hairpin queues for the selected port. */
2432 static int
2433 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2434 {
2435         queueid_t qi;
2436         struct rte_eth_hairpin_conf hairpin_conf = {
2437                 .peer_count = 1,
2438         };
2439         int i;
2440         int diag;
2441         struct rte_port *port = &ports[pi];
2442         uint16_t peer_rx_port = pi;
2443         uint16_t peer_tx_port = pi;
2444         uint32_t manual = 1;
2445         uint32_t tx_exp = hairpin_mode & 0x10;
2446
2447         if (!(hairpin_mode & 0xf)) {
2448                 peer_rx_port = pi;
2449                 peer_tx_port = pi;
2450                 manual = 0;
2451         } else if (hairpin_mode & 0x1) {
2452                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2453                                                        RTE_ETH_DEV_NO_OWNER);
2454                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2455                         peer_tx_port = rte_eth_find_next_owned_by(0,
2456                                                 RTE_ETH_DEV_NO_OWNER);
2457                 if (p_pi != RTE_MAX_ETHPORTS) {
2458                         peer_rx_port = p_pi;
2459                 } else {
2460                         uint16_t next_pi;
2461
2462                         /* Last port will be the peer RX port of the first. */
2463                         RTE_ETH_FOREACH_DEV(next_pi)
2464                                 peer_rx_port = next_pi;
2465                 }
2466                 manual = 1;
2467         } else if (hairpin_mode & 0x2) {
2468                 if (cnt_pi & 0x1) {
2469                         peer_rx_port = p_pi;
2470                 } else {
2471                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2472                                                 RTE_ETH_DEV_NO_OWNER);
2473                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2474                                 peer_rx_port = pi;
2475                 }
2476                 peer_tx_port = peer_rx_port;
2477                 manual = 1;
2478         }
2479
2480         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2481                 hairpin_conf.peers[0].port = peer_rx_port;
2482                 hairpin_conf.peers[0].queue = i + nb_rxq;
2483                 hairpin_conf.manual_bind = !!manual;
2484                 hairpin_conf.tx_explicit = !!tx_exp;
2485                 diag = rte_eth_tx_hairpin_queue_setup
2486                         (pi, qi, nb_txd, &hairpin_conf);
2487                 i++;
2488                 if (diag == 0)
2489                         continue;
2490
2491                 /* Fail to setup rx queue, return */
2492                 if (rte_atomic16_cmpset(&(port->port_status),
2493                                         RTE_PORT_HANDLING,
2494                                         RTE_PORT_STOPPED) == 0)
2495                         fprintf(stderr,
2496                                 "Port %d can not be set back to stopped\n", pi);
2497                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2498                         pi);
2499                 /* try to reconfigure queues next time */
2500                 port->need_reconfig_queues = 1;
2501                 return -1;
2502         }
2503         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2504                 hairpin_conf.peers[0].port = peer_tx_port;
2505                 hairpin_conf.peers[0].queue = i + nb_txq;
2506                 hairpin_conf.manual_bind = !!manual;
2507                 hairpin_conf.tx_explicit = !!tx_exp;
2508                 diag = rte_eth_rx_hairpin_queue_setup
2509                         (pi, qi, nb_rxd, &hairpin_conf);
2510                 i++;
2511                 if (diag == 0)
2512                         continue;
2513
2514                 /* Fail to setup rx queue, return */
2515                 if (rte_atomic16_cmpset(&(port->port_status),
2516                                         RTE_PORT_HANDLING,
2517                                         RTE_PORT_STOPPED) == 0)
2518                         fprintf(stderr,
2519                                 "Port %d can not be set back to stopped\n", pi);
2520                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2521                         pi);
2522                 /* try to reconfigure queues next time */
2523                 port->need_reconfig_queues = 1;
2524                 return -1;
2525         }
2526         return 0;
2527 }
2528
2529 /* Configure the Rx with optional split. */
2530 int
2531 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2532                uint16_t nb_rx_desc, unsigned int socket_id,
2533                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2534 {
2535         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2536         unsigned int i, mp_n;
2537         int ret;
2538
2539         if (rx_pkt_nb_segs <= 1 ||
2540             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2541                 rx_conf->rx_seg = NULL;
2542                 rx_conf->rx_nseg = 0;
2543                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2544                                              nb_rx_desc, socket_id,
2545                                              rx_conf, mp);
2546                 return ret;
2547         }
2548         for (i = 0; i < rx_pkt_nb_segs; i++) {
2549                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2550                 struct rte_mempool *mpx;
2551                 /*
2552                  * Use last valid pool for the segments with number
2553                  * exceeding the pool index.
2554                  */
2555                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2556                 mpx = mbuf_pool_find(socket_id, mp_n);
2557                 /* Handle zero as mbuf data buffer size. */
2558                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2559                                    rx_pkt_seg_lengths[i] :
2560                                    mbuf_data_size[mp_n];
2561                 rx_seg->offset = i < rx_pkt_nb_offs ?
2562                                    rx_pkt_seg_offsets[i] : 0;
2563                 rx_seg->mp = mpx ? mpx : mp;
2564         }
2565         rx_conf->rx_nseg = rx_pkt_nb_segs;
2566         rx_conf->rx_seg = rx_useg;
2567         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2568                                     socket_id, rx_conf, NULL);
2569         rx_conf->rx_seg = NULL;
2570         rx_conf->rx_nseg = 0;
2571         return ret;
2572 }
2573
2574 int
2575 start_port(portid_t pid)
2576 {
2577         int diag, need_check_link_status = -1;
2578         portid_t pi;
2579         portid_t p_pi = RTE_MAX_ETHPORTS;
2580         portid_t pl[RTE_MAX_ETHPORTS];
2581         portid_t peer_pl[RTE_MAX_ETHPORTS];
2582         uint16_t cnt_pi = 0;
2583         uint16_t cfg_pi = 0;
2584         int peer_pi;
2585         queueid_t qi;
2586         struct rte_port *port;
2587         struct rte_eth_hairpin_cap cap;
2588
2589         if (port_id_is_invalid(pid, ENABLED_WARN))
2590                 return 0;
2591
2592         RTE_ETH_FOREACH_DEV(pi) {
2593                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2594                         continue;
2595
2596                 need_check_link_status = 0;
2597                 port = &ports[pi];
2598                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2599                                                  RTE_PORT_HANDLING) == 0) {
2600                         fprintf(stderr, "Port %d is now not stopped\n", pi);
2601                         continue;
2602                 }
2603
2604                 if (port->need_reconfig > 0) {
2605                         port->need_reconfig = 0;
2606
2607                         if (flow_isolate_all) {
2608                                 int ret = port_flow_isolate(pi, 1);
2609                                 if (ret) {
2610                                         fprintf(stderr,
2611                                                 "Failed to apply isolated mode on port %d\n",
2612                                                 pi);
2613                                         return -1;
2614                                 }
2615                         }
2616                         configure_rxtx_dump_callbacks(0);
2617                         printf("Configuring Port %d (socket %u)\n", pi,
2618                                         port->socket_id);
2619                         if (nb_hairpinq > 0 &&
2620                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2621                                 fprintf(stderr,
2622                                         "Port %d doesn't support hairpin queues\n",
2623                                         pi);
2624                                 return -1;
2625                         }
2626                         /* configure port */
2627                         diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
2628                                                      nb_txq + nb_hairpinq,
2629                                                      &(port->dev_conf));
2630                         if (diag != 0) {
2631                                 if (rte_atomic16_cmpset(&(port->port_status),
2632                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2633                                         fprintf(stderr,
2634                                                 "Port %d can not be set back to stopped\n",
2635                                                 pi);
2636                                 fprintf(stderr, "Fail to configure port %d\n",
2637                                         pi);
2638                                 /* try to reconfigure port next time */
2639                                 port->need_reconfig = 1;
2640                                 return -1;
2641                         }
2642                 }
2643                 if (port->need_reconfig_queues > 0 && is_proc_primary()) {
2644                         port->need_reconfig_queues = 0;
2645                         /* setup tx queues */
2646                         for (qi = 0; qi < nb_txq; qi++) {
2647                                 if ((numa_support) &&
2648                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2649                                         diag = rte_eth_tx_queue_setup(pi, qi,
2650                                                 port->nb_tx_desc[qi],
2651                                                 txring_numa[pi],
2652                                                 &(port->tx_conf[qi]));
2653                                 else
2654                                         diag = rte_eth_tx_queue_setup(pi, qi,
2655                                                 port->nb_tx_desc[qi],
2656                                                 port->socket_id,
2657                                                 &(port->tx_conf[qi]));
2658
2659                                 if (diag == 0)
2660                                         continue;
2661
2662                                 /* Fail to setup tx queue, return */
2663                                 if (rte_atomic16_cmpset(&(port->port_status),
2664                                                         RTE_PORT_HANDLING,
2665                                                         RTE_PORT_STOPPED) == 0)
2666                                         fprintf(stderr,
2667                                                 "Port %d can not be set back to stopped\n",
2668                                                 pi);
2669                                 fprintf(stderr,
2670                                         "Fail to configure port %d tx queues\n",
2671                                         pi);
2672                                 /* try to reconfigure queues next time */
2673                                 port->need_reconfig_queues = 1;
2674                                 return -1;
2675                         }
2676                         for (qi = 0; qi < nb_rxq; qi++) {
2677                                 /* setup rx queues */
2678                                 if ((numa_support) &&
2679                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2680                                         struct rte_mempool * mp =
2681                                                 mbuf_pool_find
2682                                                         (rxring_numa[pi], 0);
2683                                         if (mp == NULL) {
2684                                                 fprintf(stderr,
2685                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2686                                                         rxring_numa[pi]);
2687                                                 return -1;
2688                                         }
2689
2690                                         diag = rx_queue_setup(pi, qi,
2691                                              port->nb_rx_desc[qi],
2692                                              rxring_numa[pi],
2693                                              &(port->rx_conf[qi]),
2694                                              mp);
2695                                 } else {
2696                                         struct rte_mempool *mp =
2697                                                 mbuf_pool_find
2698                                                         (port->socket_id, 0);
2699                                         if (mp == NULL) {
2700                                                 fprintf(stderr,
2701                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2702                                                         port->socket_id);
2703                                                 return -1;
2704                                         }
2705                                         diag = rx_queue_setup(pi, qi,
2706                                              port->nb_rx_desc[qi],
2707                                              port->socket_id,
2708                                              &(port->rx_conf[qi]),
2709                                              mp);
2710                                 }
2711                                 if (diag == 0)
2712                                         continue;
2713
2714                                 /* Fail to setup rx queue, return */
2715                                 if (rte_atomic16_cmpset(&(port->port_status),
2716                                                         RTE_PORT_HANDLING,
2717                                                         RTE_PORT_STOPPED) == 0)
2718                                         fprintf(stderr,
2719                                                 "Port %d can not be set back to stopped\n",
2720                                                 pi);
2721                                 fprintf(stderr,
2722                                         "Fail to configure port %d rx queues\n",
2723                                         pi);
2724                                 /* try to reconfigure queues next time */
2725                                 port->need_reconfig_queues = 1;
2726                                 return -1;
2727                         }
2728                         /* setup hairpin queues */
2729                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2730                                 return -1;
2731                 }
2732                 configure_rxtx_dump_callbacks(verbose_level);
2733                 if (clear_ptypes) {
2734                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2735                                         NULL, 0);
2736                         if (diag < 0)
2737                                 fprintf(stderr,
2738                                         "Port %d: Failed to disable Ptype parsing\n",
2739                                         pi);
2740                 }
2741
2742                 p_pi = pi;
2743                 cnt_pi++;
2744
2745                 /* start port */
2746                 diag = eth_dev_start_mp(pi);
2747                 if (diag < 0) {
2748                         fprintf(stderr, "Fail to start port %d: %s\n",
2749                                 pi, rte_strerror(-diag));
2750
2751                         /* Fail to setup rx queue, return */
2752                         if (rte_atomic16_cmpset(&(port->port_status),
2753                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2754                                 fprintf(stderr,
2755                                         "Port %d can not be set back to stopped\n",
2756                                         pi);
2757                         continue;
2758                 }
2759
2760                 if (rte_atomic16_cmpset(&(port->port_status),
2761                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2762                         fprintf(stderr, "Port %d can not be set into started\n",
2763                                 pi);
2764
2765                 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
2766                         printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi,
2767                                         RTE_ETHER_ADDR_BYTES(&port->eth_addr));
2768
2769                 /* at least one port started, need checking link status */
2770                 need_check_link_status = 1;
2771
2772                 pl[cfg_pi++] = pi;
2773         }
2774
2775         if (need_check_link_status == 1 && !no_link_check)
2776                 check_all_ports_link_status(RTE_PORT_ALL);
2777         else if (need_check_link_status == 0)
2778                 fprintf(stderr, "Please stop the ports first\n");
2779
2780         if (hairpin_mode & 0xf) {
2781                 uint16_t i;
2782                 int j;
2783
2784                 /* bind all started hairpin ports */
2785                 for (i = 0; i < cfg_pi; i++) {
2786                         pi = pl[i];
2787                         /* bind current Tx to all peer Rx */
2788                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2789                                                         RTE_MAX_ETHPORTS, 1);
2790                         if (peer_pi < 0)
2791                                 return peer_pi;
2792                         for (j = 0; j < peer_pi; j++) {
2793                                 if (!port_is_started(peer_pl[j]))
2794                                         continue;
2795                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2796                                 if (diag < 0) {
2797                                         fprintf(stderr,
2798                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
2799                                                 pi, peer_pl[j],
2800                                                 rte_strerror(-diag));
2801                                         return -1;
2802                                 }
2803                         }
2804                         /* bind all peer Tx to current Rx */
2805                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2806                                                         RTE_MAX_ETHPORTS, 0);
2807                         if (peer_pi < 0)
2808                                 return peer_pi;
2809                         for (j = 0; j < peer_pi; j++) {
2810                                 if (!port_is_started(peer_pl[j]))
2811                                         continue;
2812                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2813                                 if (diag < 0) {
2814                                         fprintf(stderr,
2815                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
2816                                                 peer_pl[j], pi,
2817                                                 rte_strerror(-diag));
2818                                         return -1;
2819                                 }
2820                         }
2821                 }
2822         }
2823
2824         printf("Done\n");
2825         return 0;
2826 }
2827
2828 void
2829 stop_port(portid_t pid)
2830 {
2831         portid_t pi;
2832         struct rte_port *port;
2833         int need_check_link_status = 0;
2834         portid_t peer_pl[RTE_MAX_ETHPORTS];
2835         int peer_pi;
2836
2837         if (port_id_is_invalid(pid, ENABLED_WARN))
2838                 return;
2839
2840         printf("Stopping ports...\n");
2841
2842         RTE_ETH_FOREACH_DEV(pi) {
2843                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2844                         continue;
2845
2846                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2847                         fprintf(stderr,
2848                                 "Please remove port %d from forwarding configuration.\n",
2849                                 pi);
2850                         continue;
2851                 }
2852
2853                 if (port_is_bonding_slave(pi)) {
2854                         fprintf(stderr,
2855                                 "Please remove port %d from bonded device.\n",
2856                                 pi);
2857                         continue;
2858                 }
2859
2860                 port = &ports[pi];
2861                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2862                                                 RTE_PORT_HANDLING) == 0)
2863                         continue;
2864
2865                 if (hairpin_mode & 0xf) {
2866                         int j;
2867
2868                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2869                         /* unbind all peer Tx from current Rx */
2870                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2871                                                         RTE_MAX_ETHPORTS, 0);
2872                         if (peer_pi < 0)
2873                                 continue;
2874                         for (j = 0; j < peer_pi; j++) {
2875                                 if (!port_is_started(peer_pl[j]))
2876                                         continue;
2877                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2878                         }
2879                 }
2880
2881                 if (port->flow_list)
2882                         port_flow_flush(pi);
2883
2884                 if (eth_dev_stop_mp(pi) != 0)
2885                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2886                                 pi);
2887
2888                 if (rte_atomic16_cmpset(&(port->port_status),
2889                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2890                         fprintf(stderr, "Port %d can not be set into stopped\n",
2891                                 pi);
2892                 need_check_link_status = 1;
2893         }
2894         if (need_check_link_status && !no_link_check)
2895                 check_all_ports_link_status(RTE_PORT_ALL);
2896
2897         printf("Done\n");
2898 }
2899
2900 static void
2901 remove_invalid_ports_in(portid_t *array, portid_t *total)
2902 {
2903         portid_t i;
2904         portid_t new_total = 0;
2905
2906         for (i = 0; i < *total; i++)
2907                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2908                         array[new_total] = array[i];
2909                         new_total++;
2910                 }
2911         *total = new_total;
2912 }
2913
2914 static void
2915 remove_invalid_ports(void)
2916 {
2917         remove_invalid_ports_in(ports_ids, &nb_ports);
2918         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2919         nb_cfg_ports = nb_fwd_ports;
2920 }
2921
2922 void
2923 close_port(portid_t pid)
2924 {
2925         portid_t pi;
2926         struct rte_port *port;
2927
2928         if (port_id_is_invalid(pid, ENABLED_WARN))
2929                 return;
2930
2931         printf("Closing ports...\n");
2932
2933         RTE_ETH_FOREACH_DEV(pi) {
2934                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2935                         continue;
2936
2937                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2938                         fprintf(stderr,
2939                                 "Please remove port %d from forwarding configuration.\n",
2940                                 pi);
2941                         continue;
2942                 }
2943
2944                 if (port_is_bonding_slave(pi)) {
2945                         fprintf(stderr,
2946                                 "Please remove port %d from bonded device.\n",
2947                                 pi);
2948                         continue;
2949                 }
2950
2951                 port = &ports[pi];
2952                 if (rte_atomic16_cmpset(&(port->port_status),
2953                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2954                         fprintf(stderr, "Port %d is already closed\n", pi);
2955                         continue;
2956                 }
2957
2958                 if (is_proc_primary()) {
2959                         port_flow_flush(pi);
2960                         rte_eth_dev_close(pi);
2961                 }
2962         }
2963
2964         remove_invalid_ports();
2965         printf("Done\n");
2966 }
2967
2968 void
2969 reset_port(portid_t pid)
2970 {
2971         int diag;
2972         portid_t pi;
2973         struct rte_port *port;
2974
2975         if (port_id_is_invalid(pid, ENABLED_WARN))
2976                 return;
2977
2978         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2979                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2980                 fprintf(stderr,
2981                         "Can not reset port(s), please stop port(s) first.\n");
2982                 return;
2983         }
2984
2985         printf("Resetting ports...\n");
2986
2987         RTE_ETH_FOREACH_DEV(pi) {
2988                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2989                         continue;
2990
2991                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2992                         fprintf(stderr,
2993                                 "Please remove port %d from forwarding configuration.\n",
2994                                 pi);
2995                         continue;
2996                 }
2997
2998                 if (port_is_bonding_slave(pi)) {
2999                         fprintf(stderr,
3000                                 "Please remove port %d from bonded device.\n",
3001                                 pi);
3002                         continue;
3003                 }
3004
3005                 diag = rte_eth_dev_reset(pi);
3006                 if (diag == 0) {
3007                         port = &ports[pi];
3008                         port->need_reconfig = 1;
3009                         port->need_reconfig_queues = 1;
3010                 } else {
3011                         fprintf(stderr, "Failed to reset port %d. diag=%d\n",
3012                                 pi, diag);
3013                 }
3014         }
3015
3016         printf("Done\n");
3017 }
3018
3019 void
3020 attach_port(char *identifier)
3021 {
3022         portid_t pi;
3023         struct rte_dev_iterator iterator;
3024
3025         printf("Attaching a new port...\n");
3026
3027         if (identifier == NULL) {
3028                 fprintf(stderr, "Invalid parameters are specified\n");
3029                 return;
3030         }
3031
3032         if (rte_dev_probe(identifier) < 0) {
3033                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
3034                 return;
3035         }
3036
3037         /* first attach mode: event */
3038         if (setup_on_probe_event) {
3039                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
3040                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
3041                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
3042                                         ports[pi].need_setup != 0)
3043                                 setup_attached_port(pi);
3044                 return;
3045         }
3046
3047         /* second attach mode: iterator */
3048         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
3049                 /* setup ports matching the devargs used for probing */
3050                 if (port_is_forwarding(pi))
3051                         continue; /* port was already attached before */
3052                 setup_attached_port(pi);
3053         }
3054 }
3055
3056 static void
3057 setup_attached_port(portid_t pi)
3058 {
3059         unsigned int socket_id;
3060         int ret;
3061
3062         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
3063         /* if socket_id is invalid, set to the first available socket. */
3064         if (check_socket_id(socket_id) < 0)
3065                 socket_id = socket_ids[0];
3066         reconfig(pi, socket_id);
3067         ret = rte_eth_promiscuous_enable(pi);
3068         if (ret != 0)
3069                 fprintf(stderr,
3070                         "Error during enabling promiscuous mode for port %u: %s - ignore\n",
3071                         pi, rte_strerror(-ret));
3072
3073         ports_ids[nb_ports++] = pi;
3074         fwd_ports_ids[nb_fwd_ports++] = pi;
3075         nb_cfg_ports = nb_fwd_ports;
3076         ports[pi].need_setup = 0;
3077         ports[pi].port_status = RTE_PORT_STOPPED;
3078
3079         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
3080         printf("Done\n");
3081 }
3082
3083 static void
3084 detach_device(struct rte_device *dev)
3085 {
3086         portid_t sibling;
3087
3088         if (dev == NULL) {
3089                 fprintf(stderr, "Device already removed\n");
3090                 return;
3091         }
3092
3093         printf("Removing a device...\n");
3094
3095         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3096                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3097                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3098                                 fprintf(stderr, "Port %u not stopped\n",
3099                                         sibling);
3100                                 return;
3101                         }
3102                         port_flow_flush(sibling);
3103                 }
3104         }
3105
3106         if (rte_dev_remove(dev) < 0) {
3107                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3108                 return;
3109         }
3110         remove_invalid_ports();
3111
3112         printf("Device is detached\n");
3113         printf("Now total ports is %d\n", nb_ports);
3114         printf("Done\n");
3115         return;
3116 }
3117
3118 void
3119 detach_port_device(portid_t port_id)
3120 {
3121         int ret;
3122         struct rte_eth_dev_info dev_info;
3123
3124         if (port_id_is_invalid(port_id, ENABLED_WARN))
3125                 return;
3126
3127         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3128                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3129                         fprintf(stderr, "Port not stopped\n");
3130                         return;
3131                 }
3132                 fprintf(stderr, "Port was not closed\n");
3133         }
3134
3135         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3136         if (ret != 0) {
3137                 TESTPMD_LOG(ERR,
3138                         "Failed to get device info for port %d, not detaching\n",
3139                         port_id);
3140                 return;
3141         }
3142         detach_device(dev_info.device);
3143 }
3144
3145 void
3146 detach_devargs(char *identifier)
3147 {
3148         struct rte_dev_iterator iterator;
3149         struct rte_devargs da;
3150         portid_t port_id;
3151
3152         printf("Removing a device...\n");
3153
3154         memset(&da, 0, sizeof(da));
3155         if (rte_devargs_parsef(&da, "%s", identifier)) {
3156                 fprintf(stderr, "cannot parse identifier\n");
3157                 return;
3158         }
3159
3160         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3161                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3162                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3163                                 fprintf(stderr, "Port %u not stopped\n",
3164                                         port_id);
3165                                 rte_eth_iterator_cleanup(&iterator);
3166                                 rte_devargs_reset(&da);
3167                                 return;
3168                         }
3169                         port_flow_flush(port_id);
3170                 }
3171         }
3172
3173         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3174                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3175                             da.name, da.bus->name);
3176                 rte_devargs_reset(&da);
3177                 return;
3178         }
3179
3180         remove_invalid_ports();
3181
3182         printf("Device %s is detached\n", identifier);
3183         printf("Now total ports is %d\n", nb_ports);
3184         printf("Done\n");
3185         rte_devargs_reset(&da);
3186 }
3187
3188 void
3189 pmd_test_exit(void)
3190 {
3191         portid_t pt_id;
3192         unsigned int i;
3193         int ret;
3194
3195         if (test_done == 0)
3196                 stop_packet_forwarding();
3197
3198 #ifndef RTE_EXEC_ENV_WINDOWS
3199         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3200                 if (mempools[i]) {
3201                         if (mp_alloc_type == MP_ALLOC_ANON)
3202                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3203                                                      NULL);
3204                 }
3205         }
3206 #endif
3207         if (ports != NULL) {
3208                 no_link_check = 1;
3209                 RTE_ETH_FOREACH_DEV(pt_id) {
3210                         printf("\nStopping port %d...\n", pt_id);
3211                         fflush(stdout);
3212                         stop_port(pt_id);
3213                 }
3214                 RTE_ETH_FOREACH_DEV(pt_id) {
3215                         printf("\nShutting down port %d...\n", pt_id);
3216                         fflush(stdout);
3217                         close_port(pt_id);
3218                 }
3219         }
3220
3221         if (hot_plug) {
3222                 ret = rte_dev_event_monitor_stop();
3223                 if (ret) {
3224                         RTE_LOG(ERR, EAL,
3225                                 "fail to stop device event monitor.");
3226                         return;
3227                 }
3228
3229                 ret = rte_dev_event_callback_unregister(NULL,
3230                         dev_event_callback, NULL);
3231                 if (ret < 0) {
3232                         RTE_LOG(ERR, EAL,
3233                                 "fail to unregister device event callback.\n");
3234                         return;
3235                 }
3236
3237                 ret = rte_dev_hotplug_handle_disable();
3238                 if (ret) {
3239                         RTE_LOG(ERR, EAL,
3240                                 "fail to disable hotplug handling.\n");
3241                         return;
3242                 }
3243         }
3244         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3245                 if (mempools[i])
3246                         mempool_free_mp(mempools[i]);
3247         }
3248
3249         printf("\nBye...\n");
3250 }
3251
3252 typedef void (*cmd_func_t)(void);
3253 struct pmd_test_command {
3254         const char *cmd_name;
3255         cmd_func_t cmd_func;
3256 };
3257
3258 /* Check the link status of all ports in up to 9s, and print them finally */
3259 static void
3260 check_all_ports_link_status(uint32_t port_mask)
3261 {
3262 #define CHECK_INTERVAL 100 /* 100ms */
3263 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3264         portid_t portid;
3265         uint8_t count, all_ports_up, print_flag = 0;
3266         struct rte_eth_link link;
3267         int ret;
3268         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3269
3270         printf("Checking link statuses...\n");
3271         fflush(stdout);
3272         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3273                 all_ports_up = 1;
3274                 RTE_ETH_FOREACH_DEV(portid) {
3275                         if ((port_mask & (1 << portid)) == 0)
3276                                 continue;
3277                         memset(&link, 0, sizeof(link));
3278                         ret = rte_eth_link_get_nowait(portid, &link);
3279                         if (ret < 0) {
3280                                 all_ports_up = 0;
3281                                 if (print_flag == 1)
3282                                         fprintf(stderr,
3283                                                 "Port %u link get failed: %s\n",
3284                                                 portid, rte_strerror(-ret));
3285                                 continue;
3286                         }
3287                         /* print link status if flag set */
3288                         if (print_flag == 1) {
3289                                 rte_eth_link_to_str(link_status,
3290                                         sizeof(link_status), &link);
3291                                 printf("Port %d %s\n", portid, link_status);
3292                                 continue;
3293                         }
3294                         /* clear all_ports_up flag if any link down */
3295                         if (link.link_status == ETH_LINK_DOWN) {
3296                                 all_ports_up = 0;
3297                                 break;
3298                         }
3299                 }
3300                 /* after finally printing all link status, get out */
3301                 if (print_flag == 1)
3302                         break;
3303
3304                 if (all_ports_up == 0) {
3305                         fflush(stdout);
3306                         rte_delay_ms(CHECK_INTERVAL);
3307                 }
3308
3309                 /* set the print_flag if all ports up or timeout */
3310                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3311                         print_flag = 1;
3312                 }
3313
3314                 if (lsc_interrupt)
3315                         break;
3316         }
3317 }
3318
3319 static void
3320 rmv_port_callback(void *arg)
3321 {
3322         int need_to_start = 0;
3323         int org_no_link_check = no_link_check;
3324         portid_t port_id = (intptr_t)arg;
3325         struct rte_eth_dev_info dev_info;
3326         int ret;
3327
3328         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3329
3330         if (!test_done && port_is_forwarding(port_id)) {
3331                 need_to_start = 1;
3332                 stop_packet_forwarding();
3333         }
3334         no_link_check = 1;
3335         stop_port(port_id);
3336         no_link_check = org_no_link_check;
3337
3338         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3339         if (ret != 0)
3340                 TESTPMD_LOG(ERR,
3341                         "Failed to get device info for port %d, not detaching\n",
3342                         port_id);
3343         else {
3344                 struct rte_device *device = dev_info.device;
3345                 close_port(port_id);
3346                 detach_device(device); /* might be already removed or have more ports */
3347         }
3348         if (need_to_start)
3349                 start_packet_forwarding(0);
3350 }
3351
3352 /* This function is used by the interrupt thread */
3353 static int
3354 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3355                   void *ret_param)
3356 {
3357         RTE_SET_USED(param);
3358         RTE_SET_USED(ret_param);
3359
3360         if (type >= RTE_ETH_EVENT_MAX) {
3361                 fprintf(stderr,
3362                         "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3363                         port_id, __func__, type);
3364                 fflush(stderr);
3365         } else if (event_print_mask & (UINT32_C(1) << type)) {
3366                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3367                         eth_event_desc[type]);
3368                 fflush(stdout);
3369         }
3370
3371         switch (type) {
3372         case RTE_ETH_EVENT_NEW:
3373                 ports[port_id].need_setup = 1;
3374                 ports[port_id].port_status = RTE_PORT_HANDLING;
3375                 break;
3376         case RTE_ETH_EVENT_INTR_RMV:
3377                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3378                         break;
3379                 if (rte_eal_alarm_set(100000,
3380                                 rmv_port_callback, (void *)(intptr_t)port_id))
3381                         fprintf(stderr,
3382                                 "Could not set up deferred device removal\n");
3383                 break;
3384         case RTE_ETH_EVENT_DESTROY:
3385                 ports[port_id].port_status = RTE_PORT_CLOSED;
3386                 printf("Port %u is closed\n", port_id);
3387                 break;
3388         default:
3389                 break;
3390         }
3391         return 0;
3392 }
3393
3394 static int
3395 register_eth_event_callback(void)
3396 {
3397         int ret;
3398         enum rte_eth_event_type event;
3399
3400         for (event = RTE_ETH_EVENT_UNKNOWN;
3401                         event < RTE_ETH_EVENT_MAX; event++) {
3402                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3403                                 event,
3404                                 eth_event_callback,
3405                                 NULL);
3406                 if (ret != 0) {
3407                         TESTPMD_LOG(ERR, "Failed to register callback for "
3408                                         "%s event\n", eth_event_desc[event]);
3409                         return -1;
3410                 }
3411         }
3412
3413         return 0;
3414 }
3415
3416 /* This function is used by the interrupt thread */
3417 static void
3418 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3419                              __rte_unused void *arg)
3420 {
3421         uint16_t port_id;
3422         int ret;
3423
3424         if (type >= RTE_DEV_EVENT_MAX) {
3425                 fprintf(stderr, "%s called upon invalid event %d\n",
3426                         __func__, type);
3427                 fflush(stderr);
3428         }
3429
3430         switch (type) {
3431         case RTE_DEV_EVENT_REMOVE:
3432                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3433                         device_name);
3434                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3435                 if (ret) {
3436                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3437                                 device_name);
3438                         return;
3439                 }
3440                 /*
3441                  * Because the user's callback is invoked in eal interrupt
3442                  * callback, the interrupt callback need to be finished before
3443                  * it can be unregistered when detaching device. So finish
3444                  * callback soon and use a deferred removal to detach device
3445                  * is need. It is a workaround, once the device detaching be
3446                  * moved into the eal in the future, the deferred removal could
3447                  * be deleted.
3448                  */
3449                 if (rte_eal_alarm_set(100000,
3450                                 rmv_port_callback, (void *)(intptr_t)port_id))
3451                         RTE_LOG(ERR, EAL,
3452                                 "Could not set up deferred device removal\n");
3453                 break;
3454         case RTE_DEV_EVENT_ADD:
3455                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3456                         device_name);
3457                 /* TODO: After finish kernel driver binding,
3458                  * begin to attach port.
3459                  */
3460                 break;
3461         default:
3462                 break;
3463         }
3464 }
3465
3466 static void
3467 rxtx_port_config(struct rte_port *port)
3468 {
3469         uint16_t qid;
3470         uint64_t offloads;
3471
3472         for (qid = 0; qid < nb_rxq; qid++) {
3473                 offloads = port->rx_conf[qid].offloads;
3474                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3475                 if (offloads != 0)
3476                         port->rx_conf[qid].offloads = offloads;
3477
3478                 /* Check if any Rx parameters have been passed */
3479                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3480                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3481
3482                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3483                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3484
3485                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3486                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3487
3488                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3489                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3490
3491                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3492                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3493
3494                 port->nb_rx_desc[qid] = nb_rxd;
3495         }
3496
3497         for (qid = 0; qid < nb_txq; qid++) {
3498                 offloads = port->tx_conf[qid].offloads;
3499                 port->tx_conf[qid] = port->dev_info.default_txconf;
3500                 if (offloads != 0)
3501                         port->tx_conf[qid].offloads = offloads;
3502
3503                 /* Check if any Tx parameters have been passed */
3504                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3505                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3506
3507                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3508                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3509
3510                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3511                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3512
3513                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3514                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3515
3516                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3517                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3518
3519                 port->nb_tx_desc[qid] = nb_txd;
3520         }
3521 }
3522
3523 /*
3524  * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3525  * MTU is also aligned if JUMBO_FRAME offload is not set.
3526  *
3527  * port->dev_info should be set before calling this function.
3528  *
3529  * return 0 on success, negative on error
3530  */
3531 int
3532 update_jumbo_frame_offload(portid_t portid)
3533 {
3534         struct rte_port *port = &ports[portid];
3535         uint32_t eth_overhead;
3536         uint64_t rx_offloads;
3537         int ret;
3538         bool on;
3539
3540         /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3541         if (port->dev_info.max_mtu != UINT16_MAX &&
3542             port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3543                 eth_overhead = port->dev_info.max_rx_pktlen -
3544                                 port->dev_info.max_mtu;
3545         else
3546                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3547
3548         rx_offloads = port->dev_conf.rxmode.offloads;
3549
3550         /* Default config value is 0 to use PMD specific overhead */
3551         if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3552                 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3553
3554         if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3555                 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3556                 on = false;
3557         } else {
3558                 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3559                         fprintf(stderr,
3560                                 "Frame size (%u) is not supported by port %u\n",
3561                                 port->dev_conf.rxmode.max_rx_pkt_len,
3562                                 portid);
3563                         return -1;
3564                 }
3565                 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3566                 on = true;
3567         }
3568
3569         if (rx_offloads != port->dev_conf.rxmode.offloads) {
3570                 uint16_t qid;
3571
3572                 port->dev_conf.rxmode.offloads = rx_offloads;
3573
3574                 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3575                 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3576                         if (on)
3577                                 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3578                         else
3579                                 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3580                 }
3581         }
3582
3583         /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3584          * if unset do it here
3585          */
3586         if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3587                 ret = eth_dev_set_mtu_mp(portid,
3588                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3589                 if (ret)
3590                         fprintf(stderr,
3591                                 "Failed to set MTU to %u for port %u\n",
3592                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3593                                 portid);
3594         }
3595
3596         return 0;
3597 }
3598
3599 void
3600 init_port_config(void)
3601 {
3602         portid_t pid;
3603         struct rte_port *port;
3604         int ret;
3605
3606         RTE_ETH_FOREACH_DEV(pid) {
3607                 port = &ports[pid];
3608                 port->dev_conf.fdir_conf = fdir_conf;
3609
3610                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3611                 if (ret != 0)
3612                         return;
3613
3614                 if (nb_rxq > 1) {
3615                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3616                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3617                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3618                 } else {
3619                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3620                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3621                 }
3622
3623                 if (port->dcb_flag == 0) {
3624                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3625                                 port->dev_conf.rxmode.mq_mode =
3626                                         (enum rte_eth_rx_mq_mode)
3627                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3628                         else
3629                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3630                 }
3631
3632                 rxtx_port_config(port);
3633
3634                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3635                 if (ret != 0)
3636                         return;
3637
3638 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3639                 rte_pmd_ixgbe_bypass_init(pid);
3640 #endif
3641
3642                 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3643                         port->dev_conf.intr_conf.lsc = 1;
3644                 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3645                         port->dev_conf.intr_conf.rmv = 1;
3646         }
3647 }
3648
3649 void set_port_slave_flag(portid_t slave_pid)
3650 {
3651         struct rte_port *port;
3652
3653         port = &ports[slave_pid];
3654         port->slave_flag = 1;
3655 }
3656
3657 void clear_port_slave_flag(portid_t slave_pid)
3658 {
3659         struct rte_port *port;
3660
3661         port = &ports[slave_pid];
3662         port->slave_flag = 0;
3663 }
3664
3665 uint8_t port_is_bonding_slave(portid_t slave_pid)
3666 {
3667         struct rte_port *port;
3668         struct rte_eth_dev_info dev_info;
3669         int ret;
3670
3671         port = &ports[slave_pid];
3672         ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3673         if (ret != 0) {
3674                 TESTPMD_LOG(ERR,
3675                         "Failed to get device info for port id %d,"
3676                         "cannot determine if the port is a bonded slave",
3677                         slave_pid);
3678                 return 0;
3679         }
3680         if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3681                 return 1;
3682         return 0;
3683 }
3684
3685 const uint16_t vlan_tags[] = {
3686                 0,  1,  2,  3,  4,  5,  6,  7,
3687                 8,  9, 10, 11,  12, 13, 14, 15,
3688                 16, 17, 18, 19, 20, 21, 22, 23,
3689                 24, 25, 26, 27, 28, 29, 30, 31
3690 };
3691
3692 static  int
3693 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3694                  enum dcb_mode_enable dcb_mode,
3695                  enum rte_eth_nb_tcs num_tcs,
3696                  uint8_t pfc_en)
3697 {
3698         uint8_t i;
3699         int32_t rc;
3700         struct rte_eth_rss_conf rss_conf;
3701
3702         /*
3703          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3704          * given above, and the number of traffic classes available for use.
3705          */
3706         if (dcb_mode == DCB_VT_ENABLED) {
3707                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3708                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3709                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3710                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3711
3712                 /* VMDQ+DCB RX and TX configurations */
3713                 vmdq_rx_conf->enable_default_pool = 0;
3714                 vmdq_rx_conf->default_pool = 0;
3715                 vmdq_rx_conf->nb_queue_pools =
3716                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3717                 vmdq_tx_conf->nb_queue_pools =
3718                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3719
3720                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3721                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3722                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3723                         vmdq_rx_conf->pool_map[i].pools =
3724                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3725                 }
3726                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3727                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3728                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3729                 }
3730
3731                 /* set DCB mode of RX and TX of multiple queues */
3732                 eth_conf->rxmode.mq_mode =
3733                                 (enum rte_eth_rx_mq_mode)
3734                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3735                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3736         } else {
3737                 struct rte_eth_dcb_rx_conf *rx_conf =
3738                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3739                 struct rte_eth_dcb_tx_conf *tx_conf =
3740                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3741
3742                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3743
3744                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3745                 if (rc != 0)
3746                         return rc;
3747
3748                 rx_conf->nb_tcs = num_tcs;
3749                 tx_conf->nb_tcs = num_tcs;
3750
3751                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3752                         rx_conf->dcb_tc[i] = i % num_tcs;
3753                         tx_conf->dcb_tc[i] = i % num_tcs;
3754                 }
3755
3756                 eth_conf->rxmode.mq_mode =
3757                                 (enum rte_eth_rx_mq_mode)
3758                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3759                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3760                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3761         }
3762
3763         if (pfc_en)
3764                 eth_conf->dcb_capability_en =
3765                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3766         else
3767                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3768
3769         return 0;
3770 }
3771
3772 int
3773 init_port_dcb_config(portid_t pid,
3774                      enum dcb_mode_enable dcb_mode,
3775                      enum rte_eth_nb_tcs num_tcs,
3776                      uint8_t pfc_en)
3777 {
3778         struct rte_eth_conf port_conf;
3779         struct rte_port *rte_port;
3780         int retval;
3781         uint16_t i;
3782
3783         if (num_procs > 1) {
3784                 printf("The multi-process feature doesn't support dcb.\n");
3785                 return -ENOTSUP;
3786         }
3787         rte_port = &ports[pid];
3788
3789         /* retain the original device configuration. */
3790         memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf));
3791
3792         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3793         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3794         if (retval < 0)
3795                 return retval;
3796         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3797
3798         /* re-configure the device . */
3799         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3800         if (retval < 0)
3801                 return retval;
3802
3803         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3804         if (retval != 0)
3805                 return retval;
3806
3807         /* If dev_info.vmdq_pool_base is greater than 0,
3808          * the queue id of vmdq pools is started after pf queues.
3809          */
3810         if (dcb_mode == DCB_VT_ENABLED &&
3811             rte_port->dev_info.vmdq_pool_base > 0) {
3812                 fprintf(stderr,
3813                         "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n",
3814                         pid);
3815                 return -1;
3816         }
3817
3818         /* Assume the ports in testpmd have the same dcb capability
3819          * and has the same number of rxq and txq in dcb mode
3820          */
3821         if (dcb_mode == DCB_VT_ENABLED) {
3822                 if (rte_port->dev_info.max_vfs > 0) {
3823                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3824                         nb_txq = rte_port->dev_info.nb_tx_queues;
3825                 } else {
3826                         nb_rxq = rte_port->dev_info.max_rx_queues;
3827                         nb_txq = rte_port->dev_info.max_tx_queues;
3828                 }
3829         } else {
3830                 /*if vt is disabled, use all pf queues */
3831                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3832                         nb_rxq = rte_port->dev_info.max_rx_queues;
3833                         nb_txq = rte_port->dev_info.max_tx_queues;
3834                 } else {
3835                         nb_rxq = (queueid_t)num_tcs;
3836                         nb_txq = (queueid_t)num_tcs;
3837
3838                 }
3839         }
3840         rx_free_thresh = 64;
3841
3842         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3843
3844         rxtx_port_config(rte_port);
3845         /* VLAN filter */
3846         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3847         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3848                 rx_vft_set(pid, vlan_tags[i], 1);
3849
3850         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3851         if (retval != 0)
3852                 return retval;
3853
3854         rte_port->dcb_flag = 1;
3855
3856         /* Enter DCB configuration status */
3857         dcb_config = 1;
3858
3859         return 0;
3860 }
3861
3862 static void
3863 init_port(void)
3864 {
3865         int i;
3866
3867         /* Configuration of Ethernet ports. */
3868         ports = rte_zmalloc("testpmd: ports",
3869                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3870                             RTE_CACHE_LINE_SIZE);
3871         if (ports == NULL) {
3872                 rte_exit(EXIT_FAILURE,
3873                                 "rte_zmalloc(%d struct rte_port) failed\n",
3874                                 RTE_MAX_ETHPORTS);
3875         }
3876         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3877                 LIST_INIT(&ports[i].flow_tunnel_list);
3878         /* Initialize ports NUMA structures */
3879         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3880         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3881         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3882 }
3883
3884 static void
3885 force_quit(void)
3886 {
3887         pmd_test_exit();
3888         prompt_exit();
3889 }
3890
3891 static void
3892 print_stats(void)
3893 {
3894         uint8_t i;
3895         const char clr[] = { 27, '[', '2', 'J', '\0' };
3896         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3897
3898         /* Clear screen and move to top left */
3899         printf("%s%s", clr, top_left);
3900
3901         printf("\nPort statistics ====================================");
3902         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3903                 nic_stats_display(fwd_ports_ids[i]);
3904
3905         fflush(stdout);
3906 }
3907
3908 static void
3909 signal_handler(int signum)
3910 {
3911         if (signum == SIGINT || signum == SIGTERM) {
3912                 fprintf(stderr, "\nSignal %d received, preparing to exit...\n",
3913                         signum);
3914 #ifdef RTE_LIB_PDUMP
3915                 /* uninitialize packet capture framework */
3916                 rte_pdump_uninit();
3917 #endif
3918 #ifdef RTE_LIB_LATENCYSTATS
3919                 if (latencystats_enabled != 0)
3920                         rte_latencystats_uninit();
3921 #endif
3922                 force_quit();
3923                 /* Set flag to indicate the force termination. */
3924                 f_quit = 1;
3925                 /* exit with the expected status */
3926 #ifndef RTE_EXEC_ENV_WINDOWS
3927                 signal(signum, SIG_DFL);
3928                 kill(getpid(), signum);
3929 #endif
3930         }
3931 }
3932
3933 int
3934 main(int argc, char** argv)
3935 {
3936         int diag;
3937         portid_t port_id;
3938         uint16_t count;
3939         int ret;
3940
3941         signal(SIGINT, signal_handler);
3942         signal(SIGTERM, signal_handler);
3943
3944         testpmd_logtype = rte_log_register("testpmd");
3945         if (testpmd_logtype < 0)
3946                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3947         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3948
3949         diag = rte_eal_init(argc, argv);
3950         if (diag < 0)
3951                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3952                          rte_strerror(rte_errno));
3953
3954         ret = register_eth_event_callback();
3955         if (ret != 0)
3956                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3957
3958 #ifdef RTE_LIB_PDUMP
3959         /* initialize packet capture framework */
3960         rte_pdump_init();
3961 #endif
3962
3963         count = 0;
3964         RTE_ETH_FOREACH_DEV(port_id) {
3965                 ports_ids[count] = port_id;
3966                 count++;
3967         }
3968         nb_ports = (portid_t) count;
3969         if (nb_ports == 0)
3970                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3971
3972         /* allocate port structures, and init them */
3973         init_port();
3974
3975         set_def_fwd_config();
3976         if (nb_lcores == 0)
3977                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3978                          "Check the core mask argument\n");
3979
3980         /* Bitrate/latency stats disabled by default */
3981 #ifdef RTE_LIB_BITRATESTATS
3982         bitrate_enabled = 0;
3983 #endif
3984 #ifdef RTE_LIB_LATENCYSTATS
3985         latencystats_enabled = 0;
3986 #endif
3987
3988         /* on FreeBSD, mlockall() is disabled by default */
3989 #ifdef RTE_EXEC_ENV_FREEBSD
3990         do_mlockall = 0;
3991 #else
3992         do_mlockall = 1;
3993 #endif
3994
3995         argc -= diag;
3996         argv += diag;
3997         if (argc > 1)
3998                 launch_args_parse(argc, argv);
3999
4000 #ifndef RTE_EXEC_ENV_WINDOWS
4001         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
4002                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
4003                         strerror(errno));
4004         }
4005 #endif
4006
4007         if (tx_first && interactive)
4008                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
4009                                 "interactive mode.\n");
4010
4011         if (tx_first && lsc_interrupt) {
4012                 fprintf(stderr,
4013                         "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n");
4014                 lsc_interrupt = 0;
4015         }
4016
4017         if (!nb_rxq && !nb_txq)
4018                 fprintf(stderr,
4019                         "Warning: Either rx or tx queues should be non-zero\n");
4020
4021         if (nb_rxq > 1 && nb_rxq > nb_txq)
4022                 fprintf(stderr,
4023                         "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n",
4024                         nb_rxq, nb_txq);
4025
4026         init_config();
4027
4028         if (hot_plug) {
4029                 ret = rte_dev_hotplug_handle_enable();
4030                 if (ret) {
4031                         RTE_LOG(ERR, EAL,
4032                                 "fail to enable hotplug handling.");
4033                         return -1;
4034                 }
4035
4036                 ret = rte_dev_event_monitor_start();
4037                 if (ret) {
4038                         RTE_LOG(ERR, EAL,
4039                                 "fail to start device event monitoring.");
4040                         return -1;
4041                 }
4042
4043                 ret = rte_dev_event_callback_register(NULL,
4044                         dev_event_callback, NULL);
4045                 if (ret) {
4046                         RTE_LOG(ERR, EAL,
4047                                 "fail  to register device event callback\n");
4048                         return -1;
4049                 }
4050         }
4051
4052         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
4053                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
4054
4055         /* set all ports to promiscuous mode by default */
4056         RTE_ETH_FOREACH_DEV(port_id) {
4057                 ret = rte_eth_promiscuous_enable(port_id);
4058                 if (ret != 0)
4059                         fprintf(stderr,
4060                                 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
4061                                 port_id, rte_strerror(-ret));
4062         }
4063
4064         /* Init metrics library */
4065         rte_metrics_init(rte_socket_id());
4066
4067 #ifdef RTE_LIB_LATENCYSTATS
4068         if (latencystats_enabled != 0) {
4069                 int ret = rte_latencystats_init(1, NULL);
4070                 if (ret)
4071                         fprintf(stderr,
4072                                 "Warning: latencystats init() returned error %d\n",
4073                                 ret);
4074                 fprintf(stderr, "Latencystats running on lcore %d\n",
4075                         latencystats_lcore_id);
4076         }
4077 #endif
4078
4079         /* Setup bitrate stats */
4080 #ifdef RTE_LIB_BITRATESTATS
4081         if (bitrate_enabled != 0) {
4082                 bitrate_data = rte_stats_bitrate_create();
4083                 if (bitrate_data == NULL)
4084                         rte_exit(EXIT_FAILURE,
4085                                 "Could not allocate bitrate data.\n");
4086                 rte_stats_bitrate_reg(bitrate_data);
4087         }
4088 #endif
4089
4090 #ifdef RTE_LIB_CMDLINE
4091         if (strlen(cmdline_filename) != 0)
4092                 cmdline_read_from_file(cmdline_filename);
4093
4094         if (interactive == 1) {
4095                 if (auto_start) {
4096                         printf("Start automatic packet forwarding\n");
4097                         start_packet_forwarding(0);
4098                 }
4099                 prompt();
4100                 pmd_test_exit();
4101         } else
4102 #endif
4103         {
4104                 char c;
4105                 int rc;
4106
4107                 f_quit = 0;
4108
4109                 printf("No commandline core given, start packet forwarding\n");
4110                 start_packet_forwarding(tx_first);
4111                 if (stats_period != 0) {
4112                         uint64_t prev_time = 0, cur_time, diff_time = 0;
4113                         uint64_t timer_period;
4114
4115                         /* Convert to number of cycles */
4116                         timer_period = stats_period * rte_get_timer_hz();
4117
4118                         while (f_quit == 0) {
4119                                 cur_time = rte_get_timer_cycles();
4120                                 diff_time += cur_time - prev_time;
4121
4122                                 if (diff_time >= timer_period) {
4123                                         print_stats();
4124                                         /* Reset the timer */
4125                                         diff_time = 0;
4126                                 }
4127                                 /* Sleep to avoid unnecessary checks */
4128                                 prev_time = cur_time;
4129                                 rte_delay_us_sleep(US_PER_S);
4130                         }
4131                 }
4132
4133                 printf("Press enter to exit\n");
4134                 rc = read(0, &c, 1);
4135                 pmd_test_exit();
4136                 if (rc < 0)
4137                         return 1;
4138         }
4139
4140         ret = rte_eal_cleanup();
4141         if (ret != 0)
4142                 rte_exit(EXIT_FAILURE,
4143                          "EAL cleanup failed: %s\n", strerror(-ret));
4144
4145         return EXIT_SUCCESS;
4146 }