app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #ifndef RTE_EXEC_ENV_WINDOWS
  13 #include <sys/mman.h>
  14 #endif
  15 #include <sys/types.h>
  16 #include <errno.h>
  17 #include <stdbool.h>
  18
  19 #include <sys/queue.h>
  20 #include <sys/stat.h>
  21
  22 #include <stdint.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_byteorder.h>
  29 #include <rte_log.h>
  30 #include <rte_debug.h>
  31 #include <rte_cycles.h>
  32 #include <rte_memory.h>
  33 #include <rte_memcpy.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_alarm.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_atomic.h>
  40 #include <rte_branch_prediction.h>
  41 #include <rte_mempool.h>
  42 #include <rte_malloc.h>
  43 #include <rte_mbuf.h>
  44 #include <rte_mbuf_pool_ops.h>
  45 #include <rte_interrupts.h>
  46 #include <rte_pci.h>
  47 #include <rte_ether.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_dev.h>
  50 #include <rte_string_fns.h>
  51 #ifdef RTE_NET_IXGBE
  52 #include <rte_pmd_ixgbe.h>
  53 #endif
  54 #ifdef RTE_LIB_PDUMP
  55 #include <rte_pdump.h>
  56 #endif
  57 #include <rte_flow.h>
  58 #include <rte_metrics.h>
  59 #ifdef RTE_LIB_BITRATESTATS
  60 #include <rte_bitrate.h>
  61 #endif
  62 #ifdef RTE_LIB_LATENCYSTATS
  63 #include <rte_latencystats.h>
  64 #endif
  65 #ifdef RTE_EXEC_ENV_WINDOWS
  66 #include <process.h>
  67 #endif
  68
  69 #include "testpmd.h"
  70
  71 #ifndef MAP_HUGETLB
  72 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  73 #define HUGE_FLAG (0x40000)
  74 #else
  75 #define HUGE_FLAG MAP_HUGETLB
  76 #endif
  77
  78 #ifndef MAP_HUGE_SHIFT
  79 /* older kernels (or FreeBSD) will not have this define */
  80 #define HUGE_SHIFT (26)
  81 #else
  82 #define HUGE_SHIFT MAP_HUGE_SHIFT
  83 #endif
  84
  85 #define EXTMEM_HEAP_NAME "extmem"
  86 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  87
  88 uint16_t verbose_level = 0; /**< Silent by default. */
  89 int testpmd_logtype; /**< Log type for testpmd logs */
  90
  91 /* use main core for command line ? */
  92 uint8_t interactive = 0;
  93 uint8_t auto_start = 0;
  94 uint8_t tx_first;
  95 char cmdline_filename[PATH_MAX] = {0};
  96
  97 /*
  98  * NUMA support configuration.
  99  * When set, the NUMA support attempts to dispatch the allocation of the
 100  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 101  * probed ports among the CPU sockets 0 and 1.
 102  * Otherwise, all memory is allocated from CPU socket 0.
 103  */
 104 uint8_t numa_support = 1; /**< numa enabled by default */
 105
 106 /*
 107  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 108  * not configured.
 109  */
 110 uint8_t socket_num = UMA_NO_CONFIG;
 111
 112 /*
 113  * Select mempool allocation type:
 114  * - native: use regular DPDK memory
 115  * - anon: use regular DPDK memory to create mempool, but populate using
 116  *         anonymous memory (may not be IOVA-contiguous)
 117  * - xmem: use externally allocated hugepage memory
 118  */
 119 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 120
 121 /*
 122  * Store specified sockets on which memory pool to be used by ports
 123  * is allocated.
 124  */
 125 uint8_t port_numa[RTE_MAX_ETHPORTS];
 126
 127 /*
 128  * Store specified sockets on which RX ring to be used by ports
 129  * is allocated.
 130  */
 131 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 132
 133 /*
 134  * Store specified sockets on which TX ring to be used by ports
 135  * is allocated.
 136  */
 137 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 138
 139 /*
 140  * Record the Ethernet address of peer target ports to which packets are
 141  * forwarded.
 142  * Must be instantiated with the ethernet addresses of peer traffic generator
 143  * ports.
 144  */
 145 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 146 portid_t nb_peer_eth_addrs = 0;
 147
 148 /*
 149  * Probed Target Environment.
 150  */
 151 struct rte_port *ports;        /**< For all probed ethernet ports. */
 152 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 153 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 154 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 155
 156 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 157
 158 /*
 159  * Test Forwarding Configuration.
 160  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 161  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 162  */
 163 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 164 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 165 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 166 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 167
 168 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 169 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 170
 171 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 172 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 173
 174 /*
 175  * Forwarding engines.
 176  */
 177 struct fwd_engine * fwd_engines[] = {
 178         &io_fwd_engine,
 179         &mac_fwd_engine,
 180         &mac_swap_engine,
 181         &flow_gen_engine,
 182         &rx_only_engine,
 183         &tx_only_engine,
 184         &csum_fwd_engine,
 185         &icmp_echo_engine,
 186         &noisy_vnf_engine,
 187         &five_tuple_swap_fwd_engine,
 188 #ifdef RTE_LIBRTE_IEEE1588
 189         &ieee1588_fwd_engine,
 190 #endif
 191         NULL,
 192 };
 193
 194 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 195 uint16_t mempool_flags;
 196
 197 struct fwd_config cur_fwd_config;
 198 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 199 uint32_t retry_enabled;
 200 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 201 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 202
 203 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 204 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 205         DEFAULT_MBUF_DATA_SIZE
 206 }; /**< Mbuf data space size. */
 207 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 208                                       * specified on command-line. */
 209 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 210
 211 /*
 212  * In container, it cannot terminate the process which running with 'stats-period'
 213  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 214  */
 215 uint8_t f_quit;
 216
 217 /*
 218  * Configuration of packet segments used to scatter received packets
 219  * if some of split features is configured.
 220  */
 221 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 222 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 223 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 224 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 225
 226 /*
 227  * Configuration of packet segments used by the "txonly" processing engine.
 228  */
 229 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 230 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 231         TXONLY_DEF_PACKET_LEN,
 232 };
 233 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 234
 235 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 236 /**< Split policy for packets to TX. */
 237
 238 uint8_t txonly_multi_flow;
 239 /**< Whether multiple flows are generated in TXONLY mode. */
 240
 241 uint32_t tx_pkt_times_inter;
 242 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 243
 244 uint32_t tx_pkt_times_intra;
 245 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 246
 247 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 248 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 249 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */
 250 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 251
 252 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 253 uint8_t dcb_config = 0;
 254
 255 /*
 256  * Configurable number of RX/TX queues.
 257  */
 258 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 259 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 260 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 261
 262 /*
 263  * Configurable number of RX/TX ring descriptors.
 264  * Defaults are supplied by drivers via ethdev.
 265  */
 266 #define RTE_TEST_RX_DESC_DEFAULT 0
 267 #define RTE_TEST_TX_DESC_DEFAULT 0
 268 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 269 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 270
 271 #define RTE_PMD_PARAM_UNSET -1
 272 /*
 273  * Configurable values of RX and TX ring threshold registers.
 274  */
 275
 276 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 277 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 278 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 279
 280 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 281 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 282 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 283
 284 /*
 285  * Configurable value of RX free threshold.
 286  */
 287 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 288
 289 /*
 290  * Configurable value of RX drop enable.
 291  */
 292 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 293
 294 /*
 295  * Configurable value of TX free threshold.
 296  */
 297 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 298
 299 /*
 300  * Configurable value of TX RS bit threshold.
 301  */
 302 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 303
 304 /*
 305  * Configurable value of buffered packets before sending.
 306  */
 307 uint16_t noisy_tx_sw_bufsz;
 308
 309 /*
 310  * Configurable value of packet buffer timeout.
 311  */
 312 uint16_t noisy_tx_sw_buf_flush_time;
 313
 314 /*
 315  * Configurable value for size of VNF internal memory area
 316  * used for simulating noisy neighbour behaviour
 317  */
 318 uint64_t noisy_lkup_mem_sz;
 319
 320 /*
 321  * Configurable value of number of random writes done in
 322  * VNF simulation memory area.
 323  */
 324 uint64_t noisy_lkup_num_writes;
 325
 326 /*
 327  * Configurable value of number of random reads done in
 328  * VNF simulation memory area.
 329  */
 330 uint64_t noisy_lkup_num_reads;
 331
 332 /*
 333  * Configurable value of number of random reads/writes done in
 334  * VNF simulation memory area.
 335  */
 336 uint64_t noisy_lkup_num_reads_writes;
 337
 338 /*
 339  * Receive Side Scaling (RSS) configuration.
 340  */
 341 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 342
 343 /*
 344  * Port topology configuration
 345  */
 346 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 347
 348 /*
 349  * Avoids to flush all the RX streams before starts forwarding.
 350  */
 351 uint8_t no_flush_rx = 0; /* flush by default */
 352
 353 /*
 354  * Flow API isolated mode.
 355  */
 356 uint8_t flow_isolate_all;
 357
 358 /*
 359  * Avoids to check link status when starting/stopping a port.
 360  */
 361 uint8_t no_link_check = 0; /* check by default */
 362
 363 /*
 364  * Don't automatically start all ports in interactive mode.
 365  */
 366 uint8_t no_device_start = 0;
 367
 368 /*
 369  * Enable link status change notification
 370  */
 371 uint8_t lsc_interrupt = 1; /* enabled by default */
 372
 373 /*
 374  * Enable device removal notification.
 375  */
 376 uint8_t rmv_interrupt = 1; /* enabled by default */
 377
 378 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 379
 380 /* After attach, port setup is called on event or by iterator */
 381 bool setup_on_probe_event = true;
 382
 383 /* Clear ptypes on port initialization. */
 384 uint8_t clear_ptypes = true;
 385
 386 /* Hairpin ports configuration mode. */
 387 uint16_t hairpin_mode;
 388
 389 /* Pretty printing of ethdev events */
 390 static const char * const eth_event_desc[] = {
 391         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 392         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 393         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 394         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 395         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 396         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 397         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 398         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 399         [RTE_ETH_EVENT_NEW] = "device probed",
 400         [RTE_ETH_EVENT_DESTROY] = "device released",
 401         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 402         [RTE_ETH_EVENT_MAX] = NULL,
 403 };
 404
 405 /*
 406  * Display or mask ether events
 407  * Default to all events except VF_MBOX
 408  */
 409 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 413                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 414                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 415                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 416                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 417 /*
 418  * Decide if all memory are locked for performance.
 419  */
 420 int do_mlockall = 0;
 421
 422 /*
 423  * NIC bypass mode configuration options.
 424  */
 425
 426 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 427 /* The NIC bypass watchdog timeout. */
 428 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 429 #endif
 430
 431
 432 #ifdef RTE_LIB_LATENCYSTATS
 433
 434 /*
 435  * Set when latency stats is enabled in the commandline
 436  */
 437 uint8_t latencystats_enabled;
 438
 439 /*
 440  * Lcore ID to serive latency statistics.
 441  */
 442 lcoreid_t latencystats_lcore_id = -1;
 443
 444 #endif
 445
 446 /*
 447  * Ethernet device configuration.
 448  */
 449 struct rte_eth_rxmode rx_mode = {
 450         /* Default maximum frame length.
 451          * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
 452          * in init_config().
 453          */
 454         .max_rx_pkt_len = 0,
 455 };
 456
 457 struct rte_eth_txmode tx_mode = {
 458         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 459 };
 460
 461 struct rte_fdir_conf fdir_conf = {
 462         .mode = RTE_FDIR_MODE_NONE,
 463         .pballoc = RTE_FDIR_PBALLOC_64K,
 464         .status = RTE_FDIR_REPORT_STATUS,
 465         .mask = {
 466                 .vlan_tci_mask = 0xFFEF,
 467                 .ipv4_mask     = {
 468                         .src_ip = 0xFFFFFFFF,
 469                         .dst_ip = 0xFFFFFFFF,
 470                 },
 471                 .ipv6_mask     = {
 472                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 473                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 474                 },
 475                 .src_port_mask = 0xFFFF,
 476                 .dst_port_mask = 0xFFFF,
 477                 .mac_addr_byte_mask = 0xFF,
 478                 .tunnel_type_mask = 1,
 479                 .tunnel_id_mask = 0xFFFFFFFF,
 480         },
 481         .drop_queue = 127,
 482 };
 483
 484 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 485
 486 /*
 487  * Display zero values by default for xstats
 488  */
 489 uint8_t xstats_hide_zero;
 490
 491 /*
 492  * Measure of CPU cycles disabled by default
 493  */
 494 uint8_t record_core_cycles;
 495
 496 /*
 497  * Display of RX and TX bursts disabled by default
 498  */
 499 uint8_t record_burst_stats;
 500
 501 unsigned int num_sockets = 0;
 502 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 503
 504 #ifdef RTE_LIB_BITRATESTATS
 505 /* Bitrate statistics */
 506 struct rte_stats_bitrates *bitrate_data;
 507 lcoreid_t bitrate_lcore_id;
 508 uint8_t bitrate_enabled;
 509 #endif
 510
 511 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 512 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 513
 514 /*
 515  * hexadecimal bitmask of RX mq mode can be enabled.
 516  */
 517 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 518
 519 /*
 520  * Used to set forced link speed
 521  */
 522 uint32_t eth_link_speed;
 523
 524 /* Forward function declarations */
 525 static void setup_attached_port(portid_t pi);
 526 static void check_all_ports_link_status(uint32_t port_mask);
 527 static int eth_event_callback(portid_t port_id,
 528                               enum rte_eth_event_type type,
 529                               void *param, void *ret_param);
 530 static void dev_event_callback(const char *device_name,
 531                                 enum rte_dev_event_type type,
 532                                 void *param);
 533
 534 /*
 535  * Check if all the ports are started.
 536  * If yes, return positive value. If not, return zero.
 537  */
 538 static int all_ports_started(void);
 539
 540 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 541 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 542
 543 /* Holds the registered mbuf dynamic flags names. */
 544 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 545
 546 /*
 547  * Helper function to check if socket is already discovered.
 548  * If yes, return positive value. If not, return zero.
 549  */
 550 int
 551 new_socket_id(unsigned int socket_id)
 552 {
 553         unsigned int i;
 554
 555         for (i = 0; i < num_sockets; i++) {
 556                 if (socket_ids[i] == socket_id)
 557                         return 0;
 558         }
 559         return 1;
 560 }
 561
 562 /*
 563  * Setup default configuration.
 564  */
 565 static void
 566 set_default_fwd_lcores_config(void)
 567 {
 568         unsigned int i;
 569         unsigned int nb_lc;
 570         unsigned int sock_num;
 571
 572         nb_lc = 0;
 573         for (i = 0; i < RTE_MAX_LCORE; i++) {
 574                 if (!rte_lcore_is_enabled(i))
 575                         continue;
 576                 sock_num = rte_lcore_to_socket_id(i);
 577                 if (new_socket_id(sock_num)) {
 578                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 579                                 rte_exit(EXIT_FAILURE,
 580                                          "Total sockets greater than %u\n",
 581                                          RTE_MAX_NUMA_NODES);
 582                         }
 583                         socket_ids[num_sockets++] = sock_num;
 584                 }
 585                 if (i == rte_get_main_lcore())
 586                         continue;
 587                 fwd_lcores_cpuids[nb_lc++] = i;
 588         }
 589         nb_lcores = (lcoreid_t) nb_lc;
 590         nb_cfg_lcores = nb_lcores;
 591         nb_fwd_lcores = 1;
 592 }
 593
 594 static void
 595 set_def_peer_eth_addrs(void)
 596 {
 597         portid_t i;
 598
 599         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 600                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 601                 peer_eth_addrs[i].addr_bytes[5] = i;
 602         }
 603 }
 604
 605 static void
 606 set_default_fwd_ports_config(void)
 607 {
 608         portid_t pt_id;
 609         int i = 0;
 610
 611         RTE_ETH_FOREACH_DEV(pt_id) {
 612                 fwd_ports_ids[i++] = pt_id;
 613
 614                 /* Update sockets info according to the attached device */
 615                 int socket_id = rte_eth_dev_socket_id(pt_id);
 616                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 617                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 618                                 rte_exit(EXIT_FAILURE,
 619                                          "Total sockets greater than %u\n",
 620                                          RTE_MAX_NUMA_NODES);
 621                         }
 622                         socket_ids[num_sockets++] = socket_id;
 623                 }
 624         }
 625
 626         nb_cfg_ports = nb_ports;
 627         nb_fwd_ports = nb_ports;
 628 }
 629
 630 void
 631 set_def_fwd_config(void)
 632 {
 633         set_default_fwd_lcores_config();
 634         set_def_peer_eth_addrs();
 635         set_default_fwd_ports_config();
 636 }
 637
 638 #ifndef RTE_EXEC_ENV_WINDOWS
 639 /* extremely pessimistic estimation of memory required to create a mempool */
 640 static int
 641 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 642 {
 643         unsigned int n_pages, mbuf_per_pg, leftover;
 644         uint64_t total_mem, mbuf_mem, obj_sz;
 645
 646         /* there is no good way to predict how much space the mempool will
 647          * occupy because it will allocate chunks on the fly, and some of those
 648          * will come from default DPDK memory while some will come from our
 649          * external memory, so just assume 128MB will be enough for everyone.
 650          */
 651         uint64_t hdr_mem = 128 << 20;
 652
 653         /* account for possible non-contiguousness */
 654         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 655         if (obj_sz > pgsz) {
 656                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 657                 return -1;
 658         }
 659
 660         mbuf_per_pg = pgsz / obj_sz;
 661         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 662         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 663
 664         mbuf_mem = n_pages * pgsz;
 665
 666         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 667
 668         if (total_mem > SIZE_MAX) {
 669                 TESTPMD_LOG(ERR, "Memory size too big\n");
 670                 return -1;
 671         }
 672         *out = (size_t)total_mem;
 673
 674         return 0;
 675 }
 676
 677 static int
 678 pagesz_flags(uint64_t page_sz)
 679 {
 680         /* as per mmap() manpage, all page sizes are log2 of page size
 681          * shifted by MAP_HUGE_SHIFT
 682          */
 683         int log2 = rte_log2_u64(page_sz);
 684
 685         return (log2 << HUGE_SHIFT);
 686 }
 687
 688 static void *
 689 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 690 {
 691         void *addr;
 692         int flags;
 693
 694         /* allocate anonymous hugepages */
 695         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 696         if (huge)
 697                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 698
 699         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 700         if (addr == MAP_FAILED)
 701                 return NULL;
 702
 703         return addr;
 704 }
 705
 706 struct extmem_param {
 707         void *addr;
 708         size_t len;
 709         size_t pgsz;
 710         rte_iova_t *iova_table;
 711         unsigned int iova_table_len;
 712 };
 713
 714 static int
 715 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 716                 bool huge)
 717 {
 718         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 719                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 720         unsigned int cur_page, n_pages, pgsz_idx;
 721         size_t mem_sz, cur_pgsz;
 722         rte_iova_t *iovas = NULL;
 723         void *addr;
 724         int ret;
 725
 726         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 727                 /* skip anything that is too big */
 728                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 729                         continue;
 730
 731                 cur_pgsz = pgsizes[pgsz_idx];
 732
 733                 /* if we were told not to allocate hugepages, override */
 734                 if (!huge)
 735                         cur_pgsz = sysconf(_SC_PAGESIZE);
 736
 737                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 738                 if (ret < 0) {
 739                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 740                         return -1;
 741                 }
 742
 743                 /* allocate our memory */
 744                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 745
 746                 /* if we couldn't allocate memory with a specified page size,
 747                  * that doesn't mean we can't do it with other page sizes, so
 748                  * try another one.
 749                  */
 750                 if (addr == NULL)
 751                         continue;
 752
 753                 /* store IOVA addresses for every page in this memory area */
 754                 n_pages = mem_sz / cur_pgsz;
 755
 756                 iovas = malloc(sizeof(*iovas) * n_pages);
 757
 758                 if (iovas == NULL) {
 759                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 760                         goto fail;
 761                 }
 762                 /* lock memory if it's not huge pages */
 763                 if (!huge)
 764                         mlock(addr, mem_sz);
 765
 766                 /* populate IOVA addresses */
 767                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 768                         rte_iova_t iova;
 769                         size_t offset;
 770                         void *cur;
 771
 772                         offset = cur_pgsz * cur_page;
 773                         cur = RTE_PTR_ADD(addr, offset);
 774
 775                         /* touch the page before getting its IOVA */
 776                         *(volatile char *)cur = 0;
 777
 778                         iova = rte_mem_virt2iova(cur);
 779
 780                         iovas[cur_page] = iova;
 781                 }
 782
 783                 break;
 784         }
 785         /* if we couldn't allocate anything */
 786         if (iovas == NULL)
 787                 return -1;
 788
 789         param->addr = addr;
 790         param->len = mem_sz;
 791         param->pgsz = cur_pgsz;
 792         param->iova_table = iovas;
 793         param->iova_table_len = n_pages;
 794
 795         return 0;
 796 fail:
 797         if (iovas)
 798                 free(iovas);
 799         if (addr)
 800                 munmap(addr, mem_sz);
 801
 802         return -1;
 803 }
 804
 805 static int
 806 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 807 {
 808         struct extmem_param param;
 809         int socket_id, ret;
 810
 811         memset(&param, 0, sizeof(param));
 812
 813         /* check if our heap exists */
 814         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 815         if (socket_id < 0) {
 816                 /* create our heap */
 817                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 818                 if (ret < 0) {
 819                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 820                         return -1;
 821                 }
 822         }
 823
 824         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 825         if (ret < 0) {
 826                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 827                 return -1;
 828         }
 829
 830         /* we now have a valid memory area, so add it to heap */
 831         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 832                         param.addr, param.len, param.iova_table,
 833                         param.iova_table_len, param.pgsz);
 834
 835         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 836
 837         /* not needed any more */
 838         free(param.iova_table);
 839
 840         if (ret < 0) {
 841                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 842                 munmap(param.addr, param.len);
 843                 return -1;
 844         }
 845
 846         /* success */
 847
 848         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 849                         param.len >> 20);
 850
 851         return 0;
 852 }
 853 static void
 854 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 855              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 856 {
 857         uint16_t pid = 0;
 858         int ret;
 859
 860         RTE_ETH_FOREACH_DEV(pid) {
 861                 struct rte_eth_dev_info dev_info;
 862
 863                 ret = eth_dev_info_get_print_err(pid, &dev_info);
 864                 if (ret != 0) {
 865                         TESTPMD_LOG(DEBUG,
 866                                     "unable to get device info for port %d on addr 0x%p,"
 867                                     "mempool unmapping will not be performed\n",
 868                                     pid, memhdr->addr);
 869                         continue;
 870                 }
 871
 872                 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
 873                 if (ret) {
 874                         TESTPMD_LOG(DEBUG,
 875                                     "unable to DMA unmap addr 0x%p "
 876                                     "for device %s\n",
 877                                     memhdr->addr, dev_info.device->name);
 878                 }
 879         }
 880         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 881         if (ret) {
 882                 TESTPMD_LOG(DEBUG,
 883                             "unable to un-register addr 0x%p\n", memhdr->addr);
 884         }
 885 }
 886
 887 static void
 888 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 889            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 890 {
 891         uint16_t pid = 0;
 892         size_t page_size = sysconf(_SC_PAGESIZE);
 893         int ret;
 894
 895         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 896                                   page_size);
 897         if (ret) {
 898                 TESTPMD_LOG(DEBUG,
 899                             "unable to register addr 0x%p\n", memhdr->addr);
 900                 return;
 901         }
 902         RTE_ETH_FOREACH_DEV(pid) {
 903                 struct rte_eth_dev_info dev_info;
 904
 905                 ret = eth_dev_info_get_print_err(pid, &dev_info);
 906                 if (ret != 0) {
 907                         TESTPMD_LOG(DEBUG,
 908                                     "unable to get device info for port %d on addr 0x%p,"
 909                                     "mempool mapping will not be performed\n",
 910                                     pid, memhdr->addr);
 911                         continue;
 912                 }
 913                 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
 914                 if (ret) {
 915                         TESTPMD_LOG(DEBUG,
 916                                     "unable to DMA map addr 0x%p "
 917                                     "for device %s\n",
 918                                     memhdr->addr, dev_info.device->name);
 919                 }
 920         }
 921 }
 922 #endif
 923
 924 static unsigned int
 925 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 926             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 927 {
 928         struct rte_pktmbuf_extmem *xmem;
 929         unsigned int ext_num, zone_num, elt_num;
 930         uint16_t elt_size;
 931
 932         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 933         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 934         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 935
 936         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 937         if (xmem == NULL) {
 938                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 939                                  "external buffer descriptors\n");
 940                 *ext_mem = NULL;
 941                 return 0;
 942         }
 943         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 944                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 945                 const struct rte_memzone *mz;
 946                 char mz_name[RTE_MEMZONE_NAMESIZE];
 947                 int ret;
 948
 949                 ret = snprintf(mz_name, sizeof(mz_name),
 950                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 951                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 952                         errno = ENAMETOOLONG;
 953                         ext_num = 0;
 954                         break;
 955                 }
 956                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 957                                                  socket_id,
 958                                                  RTE_MEMZONE_IOVA_CONTIG |
 959                                                  RTE_MEMZONE_1GB |
 960                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 961                                                  EXTBUF_ZONE_SIZE);
 962                 if (mz == NULL) {
 963                         /*
 964                          * The caller exits on external buffer creation
 965                          * error, so there is no need to free memzones.
 966                          */
 967                         errno = ENOMEM;
 968                         ext_num = 0;
 969                         break;
 970                 }
 971                 xseg->buf_ptr = mz->addr;
 972                 xseg->buf_iova = mz->iova;
 973                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 974                 xseg->elt_size = elt_size;
 975         }
 976         if (ext_num == 0 && xmem != NULL) {
 977                 free(xmem);
 978                 xmem = NULL;
 979         }
 980         *ext_mem = xmem;
 981         return ext_num;
 982 }
 983
 984 /*
 985  * Configuration initialisation done once at init time.
 986  */
 987 static struct rte_mempool *
 988 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 989                  unsigned int socket_id, uint16_t size_idx)
 990 {
 991         char pool_name[RTE_MEMPOOL_NAMESIZE];
 992         struct rte_mempool *rte_mp = NULL;
 993 #ifndef RTE_EXEC_ENV_WINDOWS
 994         uint32_t mb_size;
 995
 996         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 997 #endif
 998         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 999
1000         TESTPMD_LOG(INFO,
1001                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1002                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1003
1004         switch (mp_alloc_type) {
1005         case MP_ALLOC_NATIVE:
1006                 {
1007                         /* wrapper to rte_mempool_create() */
1008                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1009                                         rte_mbuf_best_mempool_ops());
1010                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1011                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1012                         break;
1013                 }
1014 #ifndef RTE_EXEC_ENV_WINDOWS
1015         case MP_ALLOC_ANON:
1016                 {
1017                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1018                                 mb_size, (unsigned int) mb_mempool_cache,
1019                                 sizeof(struct rte_pktmbuf_pool_private),
1020                                 socket_id, mempool_flags);
1021                         if (rte_mp == NULL)
1022                                 goto err;
1023
1024                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1025                                 rte_mempool_free(rte_mp);
1026                                 rte_mp = NULL;
1027                                 goto err;
1028                         }
1029                         rte_pktmbuf_pool_init(rte_mp, NULL);
1030                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1031                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1032                         break;
1033                 }
1034         case MP_ALLOC_XMEM:
1035         case MP_ALLOC_XMEM_HUGE:
1036                 {
1037                         int heap_socket;
1038                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1039
1040                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1041                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1042
1043                         heap_socket =
1044                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1045                         if (heap_socket < 0)
1046                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1047
1048                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1049                                         rte_mbuf_best_mempool_ops());
1050                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1051                                         mb_mempool_cache, 0, mbuf_seg_size,
1052                                         heap_socket);
1053                         break;
1054                 }
1055 #endif
1056         case MP_ALLOC_XBUF:
1057                 {
1058                         struct rte_pktmbuf_extmem *ext_mem;
1059                         unsigned int ext_num;
1060
1061                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1062                                                socket_id, pool_name, &ext_mem);
1063                         if (ext_num == 0)
1064                                 rte_exit(EXIT_FAILURE,
1065                                          "Can't create pinned data buffers\n");
1066
1067                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1068                                         rte_mbuf_best_mempool_ops());
1069                         rte_mp = rte_pktmbuf_pool_create_extbuf
1070                                         (pool_name, nb_mbuf, mb_mempool_cache,
1071                                          0, mbuf_seg_size, socket_id,
1072                                          ext_mem, ext_num);
1073                         free(ext_mem);
1074                         break;
1075                 }
1076         default:
1077                 {
1078                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1079                 }
1080         }
1081
1082 #ifndef RTE_EXEC_ENV_WINDOWS
1083 err:
1084 #endif
1085         if (rte_mp == NULL) {
1086                 rte_exit(EXIT_FAILURE,
1087                         "Creation of mbuf pool for socket %u failed: %s\n",
1088                         socket_id, rte_strerror(rte_errno));
1089         } else if (verbose_level > 0) {
1090                 rte_mempool_dump(stdout, rte_mp);
1091         }
1092         return rte_mp;
1093 }
1094
1095 /*
1096  * Check given socket id is valid or not with NUMA mode,
1097  * if valid, return 0, else return -1
1098  */
1099 static int
1100 check_socket_id(const unsigned int socket_id)
1101 {
1102         static int warning_once = 0;
1103
1104         if (new_socket_id(socket_id)) {
1105                 if (!warning_once && numa_support)
1106                         fprintf(stderr,
1107                                 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
1108                 warning_once = 1;
1109                 return -1;
1110         }
1111         return 0;
1112 }
1113
1114 /*
1115  * Get the allowed maximum number of RX queues.
1116  * *pid return the port id which has minimal value of
1117  * max_rx_queues in all ports.
1118  */
1119 queueid_t
1120 get_allowed_max_nb_rxq(portid_t *pid)
1121 {
1122         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1123         bool max_rxq_valid = false;
1124         portid_t pi;
1125         struct rte_eth_dev_info dev_info;
1126
1127         RTE_ETH_FOREACH_DEV(pi) {
1128                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1129                         continue;
1130
1131                 max_rxq_valid = true;
1132                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1133                         allowed_max_rxq = dev_info.max_rx_queues;
1134                         *pid = pi;
1135                 }
1136         }
1137         return max_rxq_valid ? allowed_max_rxq : 0;
1138 }
1139
1140 /*
1141  * Check input rxq is valid or not.
1142  * If input rxq is not greater than any of maximum number
1143  * of RX queues of all ports, it is valid.
1144  * if valid, return 0, else return -1
1145  */
1146 int
1147 check_nb_rxq(queueid_t rxq)
1148 {
1149         queueid_t allowed_max_rxq;
1150         portid_t pid = 0;
1151
1152         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1153         if (rxq > allowed_max_rxq) {
1154                 fprintf(stderr,
1155                         "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
1156                         rxq, allowed_max_rxq, pid);
1157                 return -1;
1158         }
1159         return 0;
1160 }
1161
1162 /*
1163  * Get the allowed maximum number of TX queues.
1164  * *pid return the port id which has minimal value of
1165  * max_tx_queues in all ports.
1166  */
1167 queueid_t
1168 get_allowed_max_nb_txq(portid_t *pid)
1169 {
1170         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1171         bool max_txq_valid = false;
1172         portid_t pi;
1173         struct rte_eth_dev_info dev_info;
1174
1175         RTE_ETH_FOREACH_DEV(pi) {
1176                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1177                         continue;
1178
1179                 max_txq_valid = true;
1180                 if (dev_info.max_tx_queues < allowed_max_txq) {
1181                         allowed_max_txq = dev_info.max_tx_queues;
1182                         *pid = pi;
1183                 }
1184         }
1185         return max_txq_valid ? allowed_max_txq : 0;
1186 }
1187
1188 /*
1189  * Check input txq is valid or not.
1190  * If input txq is not greater than any of maximum number
1191  * of TX queues of all ports, it is valid.
1192  * if valid, return 0, else return -1
1193  */
1194 int
1195 check_nb_txq(queueid_t txq)
1196 {
1197         queueid_t allowed_max_txq;
1198         portid_t pid = 0;
1199
1200         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1201         if (txq > allowed_max_txq) {
1202                 fprintf(stderr,
1203                         "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n",
1204                         txq, allowed_max_txq, pid);
1205                 return -1;
1206         }
1207         return 0;
1208 }
1209
1210 /*
1211  * Get the allowed maximum number of RXDs of every rx queue.
1212  * *pid return the port id which has minimal value of
1213  * max_rxd in all queues of all ports.
1214  */
1215 static uint16_t
1216 get_allowed_max_nb_rxd(portid_t *pid)
1217 {
1218         uint16_t allowed_max_rxd = UINT16_MAX;
1219         portid_t pi;
1220         struct rte_eth_dev_info dev_info;
1221
1222         RTE_ETH_FOREACH_DEV(pi) {
1223                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1224                         continue;
1225
1226                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1227                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1228                         *pid = pi;
1229                 }
1230         }
1231         return allowed_max_rxd;
1232 }
1233
1234 /*
1235  * Get the allowed minimal number of RXDs of every rx queue.
1236  * *pid return the port id which has minimal value of
1237  * min_rxd in all queues of all ports.
1238  */
1239 static uint16_t
1240 get_allowed_min_nb_rxd(portid_t *pid)
1241 {
1242         uint16_t allowed_min_rxd = 0;
1243         portid_t pi;
1244         struct rte_eth_dev_info dev_info;
1245
1246         RTE_ETH_FOREACH_DEV(pi) {
1247                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1248                         continue;
1249
1250                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1251                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1252                         *pid = pi;
1253                 }
1254         }
1255
1256         return allowed_min_rxd;
1257 }
1258
1259 /*
1260  * Check input rxd is valid or not.
1261  * If input rxd is not greater than any of maximum number
1262  * of RXDs of every Rx queues and is not less than any of
1263  * minimal number of RXDs of every Rx queues, it is valid.
1264  * if valid, return 0, else return -1
1265  */
1266 int
1267 check_nb_rxd(queueid_t rxd)
1268 {
1269         uint16_t allowed_max_rxd;
1270         uint16_t allowed_min_rxd;
1271         portid_t pid = 0;
1272
1273         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1274         if (rxd > allowed_max_rxd) {
1275                 fprintf(stderr,
1276                         "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n",
1277                         rxd, allowed_max_rxd, pid);
1278                 return -1;
1279         }
1280
1281         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1282         if (rxd < allowed_min_rxd) {
1283                 fprintf(stderr,
1284                         "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n",
1285                         rxd, allowed_min_rxd, pid);
1286                 return -1;
1287         }
1288
1289         return 0;
1290 }
1291
1292 /*
1293  * Get the allowed maximum number of TXDs of every rx queues.
1294  * *pid return the port id which has minimal value of
1295  * max_txd in every tx queue.
1296  */
1297 static uint16_t
1298 get_allowed_max_nb_txd(portid_t *pid)
1299 {
1300         uint16_t allowed_max_txd = UINT16_MAX;
1301         portid_t pi;
1302         struct rte_eth_dev_info dev_info;
1303
1304         RTE_ETH_FOREACH_DEV(pi) {
1305                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1306                         continue;
1307
1308                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1309                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1310                         *pid = pi;
1311                 }
1312         }
1313         return allowed_max_txd;
1314 }
1315
1316 /*
1317  * Get the allowed maximum number of TXDs of every tx queues.
1318  * *pid return the port id which has minimal value of
1319  * min_txd in every tx queue.
1320  */
1321 static uint16_t
1322 get_allowed_min_nb_txd(portid_t *pid)
1323 {
1324         uint16_t allowed_min_txd = 0;
1325         portid_t pi;
1326         struct rte_eth_dev_info dev_info;
1327
1328         RTE_ETH_FOREACH_DEV(pi) {
1329                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1330                         continue;
1331
1332                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1333                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1334                         *pid = pi;
1335                 }
1336         }
1337
1338         return allowed_min_txd;
1339 }
1340
1341 /*
1342  * Check input txd is valid or not.
1343  * If input txd is not greater than any of maximum number
1344  * of TXDs of every Rx queues, it is valid.
1345  * if valid, return 0, else return -1
1346  */
1347 int
1348 check_nb_txd(queueid_t txd)
1349 {
1350         uint16_t allowed_max_txd;
1351         uint16_t allowed_min_txd;
1352         portid_t pid = 0;
1353
1354         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1355         if (txd > allowed_max_txd) {
1356                 fprintf(stderr,
1357                         "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n",
1358                         txd, allowed_max_txd, pid);
1359                 return -1;
1360         }
1361
1362         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1363         if (txd < allowed_min_txd) {
1364                 fprintf(stderr,
1365                         "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n",
1366                         txd, allowed_min_txd, pid);
1367                 return -1;
1368         }
1369         return 0;
1370 }
1371
1372
1373 /*
1374  * Get the allowed maximum number of hairpin queues.
1375  * *pid return the port id which has minimal value of
1376  * max_hairpin_queues in all ports.
1377  */
1378 queueid_t
1379 get_allowed_max_nb_hairpinq(portid_t *pid)
1380 {
1381         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1382         portid_t pi;
1383         struct rte_eth_hairpin_cap cap;
1384
1385         RTE_ETH_FOREACH_DEV(pi) {
1386                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1387                         *pid = pi;
1388                         return 0;
1389                 }
1390                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1391                         allowed_max_hairpinq = cap.max_nb_queues;
1392                         *pid = pi;
1393                 }
1394         }
1395         return allowed_max_hairpinq;
1396 }
1397
1398 /*
1399  * Check input hairpin is valid or not.
1400  * If input hairpin is not greater than any of maximum number
1401  * of hairpin queues of all ports, it is valid.
1402  * if valid, return 0, else return -1
1403  */
1404 int
1405 check_nb_hairpinq(queueid_t hairpinq)
1406 {
1407         queueid_t allowed_max_hairpinq;
1408         portid_t pid = 0;
1409
1410         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1411         if (hairpinq > allowed_max_hairpinq) {
1412                 fprintf(stderr,
1413                         "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n",
1414                         hairpinq, allowed_max_hairpinq, pid);
1415                 return -1;
1416         }
1417         return 0;
1418 }
1419
1420 static void
1421 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1422 {
1423         struct rte_port *port = &ports[pid];
1424         uint16_t data_size;
1425         int ret;
1426         int i;
1427
1428         port->dev_conf.txmode = tx_mode;
1429         port->dev_conf.rxmode = rx_mode;
1430
1431         ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1432         if (ret != 0)
1433                 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1434
1435         ret = update_jumbo_frame_offload(pid);
1436         if (ret != 0)
1437                 fprintf(stderr,
1438                         "Updating jumbo frame offload failed for port %u\n",
1439                         pid);
1440
1441         if (!(port->dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1442                 port->dev_conf.txmode.offloads &=
1443                         ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1444
1445         /* Apply Rx offloads configuration */
1446         for (i = 0; i < port->dev_info.max_rx_queues; i++)
1447                 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads;
1448         /* Apply Tx offloads configuration */
1449         for (i = 0; i < port->dev_info.max_tx_queues; i++)
1450                 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads;
1451
1452         if (eth_link_speed)
1453                 port->dev_conf.link_speeds = eth_link_speed;
1454
1455         /* set flag to initialize port/queue */
1456         port->need_reconfig = 1;
1457         port->need_reconfig_queues = 1;
1458         port->socket_id = socket_id;
1459         port->tx_metadata = 0;
1460
1461         /*
1462          * Check for maximum number of segments per MTU.
1463          * Accordingly update the mbuf data size.
1464          */
1465         if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1466             port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1467                 data_size = rx_mode.max_rx_pkt_len /
1468                         port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1469
1470                 if ((data_size + RTE_PKTMBUF_HEADROOM) > mbuf_data_size[0]) {
1471                         mbuf_data_size[0] = data_size + RTE_PKTMBUF_HEADROOM;
1472                         TESTPMD_LOG(WARNING,
1473                                     "Configured mbuf size of the first segment %hu\n",
1474                                     mbuf_data_size[0]);
1475                 }
1476         }
1477 }
1478
1479 static void
1480 init_config(void)
1481 {
1482         portid_t pid;
1483         struct rte_mempool *mbp;
1484         unsigned int nb_mbuf_per_pool;
1485         lcoreid_t  lc_id;
1486         struct rte_gro_param gro_param;
1487         uint32_t gso_types;
1488
1489         /* Configuration of logical cores. */
1490         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1491                                 sizeof(struct fwd_lcore *) * nb_lcores,
1492                                 RTE_CACHE_LINE_SIZE);
1493         if (fwd_lcores == NULL) {
1494                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1495                                                         "failed\n", nb_lcores);
1496         }
1497         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1498                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1499                                                sizeof(struct fwd_lcore),
1500                                                RTE_CACHE_LINE_SIZE);
1501                 if (fwd_lcores[lc_id] == NULL) {
1502                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1503                                                                 "failed\n");
1504                 }
1505                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1506         }
1507
1508         RTE_ETH_FOREACH_DEV(pid) {
1509                 uint32_t socket_id;
1510
1511                 if (numa_support) {
1512                         socket_id = port_numa[pid];
1513                         if (port_numa[pid] == NUMA_NO_CONFIG) {
1514                                 socket_id = rte_eth_dev_socket_id(pid);
1515
1516                                 /*
1517                                  * if socket_id is invalid,
1518                                  * set to the first available socket.
1519                                  */
1520                                 if (check_socket_id(socket_id) < 0)
1521                                         socket_id = socket_ids[0];
1522                         }
1523                 } else {
1524                         socket_id = (socket_num == UMA_NO_CONFIG) ?
1525                                     0 : socket_num;
1526                 }
1527                 /* Apply default TxRx configuration for all ports */
1528                 init_config_port_offloads(pid, socket_id);
1529         }
1530         /*
1531          * Create pools of mbuf.
1532          * If NUMA support is disabled, create a single pool of mbuf in
1533          * socket 0 memory by default.
1534          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1535          *
1536          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1537          * nb_txd can be configured at run time.
1538          */
1539         if (param_total_num_mbufs)
1540                 nb_mbuf_per_pool = param_total_num_mbufs;
1541         else {
1542                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1543                         (nb_lcores * mb_mempool_cache) +
1544                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1545                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1546         }
1547
1548         if (numa_support) {
1549                 uint8_t i, j;
1550
1551                 for (i = 0; i < num_sockets; i++)
1552                         for (j = 0; j < mbuf_data_size_n; j++)
1553                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1554                                         mbuf_pool_create(mbuf_data_size[j],
1555                                                           nb_mbuf_per_pool,
1556                                                           socket_ids[i], j);
1557         } else {
1558                 uint8_t i;
1559
1560                 for (i = 0; i < mbuf_data_size_n; i++)
1561                         mempools[i] = mbuf_pool_create
1562                                         (mbuf_data_size[i],
1563                                          nb_mbuf_per_pool,
1564                                          socket_num == UMA_NO_CONFIG ?
1565                                          0 : socket_num, i);
1566         }
1567
1568         init_port_config();
1569
1570         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1571                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1572         /*
1573          * Records which Mbuf pool to use by each logical core, if needed.
1574          */
1575         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1576                 mbp = mbuf_pool_find(
1577                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1578
1579                 if (mbp == NULL)
1580                         mbp = mbuf_pool_find(0, 0);
1581                 fwd_lcores[lc_id]->mbp = mbp;
1582                 /* initialize GSO context */
1583                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1584                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1585                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1586                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1587                         RTE_ETHER_CRC_LEN;
1588                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1589         }
1590
1591         fwd_config_setup();
1592
1593         /* create a gro context for each lcore */
1594         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1595         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1596         gro_param.max_item_per_flow = MAX_PKT_BURST;
1597         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1598                 gro_param.socket_id = rte_lcore_to_socket_id(
1599                                 fwd_lcores_cpuids[lc_id]);
1600                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1601                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1602                         rte_exit(EXIT_FAILURE,
1603                                         "rte_gro_ctx_create() failed\n");
1604                 }
1605         }
1606 }
1607
1608
1609 void
1610 reconfig(portid_t new_port_id, unsigned socket_id)
1611 {
1612         /* Reconfiguration of Ethernet ports. */
1613         init_config_port_offloads(new_port_id, socket_id);
1614         init_port_config();
1615 }
1616
1617
1618 int
1619 init_fwd_streams(void)
1620 {
1621         portid_t pid;
1622         struct rte_port *port;
1623         streamid_t sm_id, nb_fwd_streams_new;
1624         queueid_t q;
1625
1626         /* set socket id according to numa or not */
1627         RTE_ETH_FOREACH_DEV(pid) {
1628                 port = &ports[pid];
1629                 if (nb_rxq > port->dev_info.max_rx_queues) {
1630                         fprintf(stderr,
1631                                 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n",
1632                                 nb_rxq, port->dev_info.max_rx_queues);
1633                         return -1;
1634                 }
1635                 if (nb_txq > port->dev_info.max_tx_queues) {
1636                         fprintf(stderr,
1637                                 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n",
1638                                 nb_txq, port->dev_info.max_tx_queues);
1639                         return -1;
1640                 }
1641                 if (numa_support) {
1642                         if (port_numa[pid] != NUMA_NO_CONFIG)
1643                                 port->socket_id = port_numa[pid];
1644                         else {
1645                                 port->socket_id = rte_eth_dev_socket_id(pid);
1646
1647                                 /*
1648                                  * if socket_id is invalid,
1649                                  * set to the first available socket.
1650                                  */
1651                                 if (check_socket_id(port->socket_id) < 0)
1652                                         port->socket_id = socket_ids[0];
1653                         }
1654                 }
1655                 else {
1656                         if (socket_num == UMA_NO_CONFIG)
1657                                 port->socket_id = 0;
1658                         else
1659                                 port->socket_id = socket_num;
1660                 }
1661         }
1662
1663         q = RTE_MAX(nb_rxq, nb_txq);
1664         if (q == 0) {
1665                 fprintf(stderr,
1666                         "Fail: Cannot allocate fwd streams as number of queues is 0\n");
1667                 return -1;
1668         }
1669         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1670         if (nb_fwd_streams_new == nb_fwd_streams)
1671                 return 0;
1672         /* clear the old */
1673         if (fwd_streams != NULL) {
1674                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1675                         if (fwd_streams[sm_id] == NULL)
1676                                 continue;
1677                         rte_free(fwd_streams[sm_id]);
1678                         fwd_streams[sm_id] = NULL;
1679                 }
1680                 rte_free(fwd_streams);
1681                 fwd_streams = NULL;
1682         }
1683
1684         /* init new */
1685         nb_fwd_streams = nb_fwd_streams_new;
1686         if (nb_fwd_streams) {
1687                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1688                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1689                         RTE_CACHE_LINE_SIZE);
1690                 if (fwd_streams == NULL)
1691                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1692                                  " (struct fwd_stream *)) failed\n",
1693                                  nb_fwd_streams);
1694
1695                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1696                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1697                                 " struct fwd_stream", sizeof(struct fwd_stream),
1698                                 RTE_CACHE_LINE_SIZE);
1699                         if (fwd_streams[sm_id] == NULL)
1700                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1701                                          "(struct fwd_stream) failed\n");
1702                 }
1703         }
1704
1705         return 0;
1706 }
1707
1708 static void
1709 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1710 {
1711         uint64_t total_burst, sburst;
1712         uint64_t nb_burst;
1713         uint64_t burst_stats[4];
1714         uint16_t pktnb_stats[4];
1715         uint16_t nb_pkt;
1716         int burst_percent[4], sburstp;
1717         int i;
1718
1719         /*
1720          * First compute the total number of packet bursts and the
1721          * two highest numbers of bursts of the same number of packets.
1722          */
1723         memset(&burst_stats, 0x0, sizeof(burst_stats));
1724         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1725
1726         /* Show stats for 0 burst size always */
1727         total_burst = pbs->pkt_burst_spread[0];
1728         burst_stats[0] = pbs->pkt_burst_spread[0];
1729         pktnb_stats[0] = 0;
1730
1731         /* Find the next 2 burst sizes with highest occurrences. */
1732         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1733                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1734
1735                 if (nb_burst == 0)
1736                         continue;
1737
1738                 total_burst += nb_burst;
1739
1740                 if (nb_burst > burst_stats[1]) {
1741                         burst_stats[2] = burst_stats[1];
1742                         pktnb_stats[2] = pktnb_stats[1];
1743                         burst_stats[1] = nb_burst;
1744                         pktnb_stats[1] = nb_pkt;
1745                 } else if (nb_burst > burst_stats[2]) {
1746                         burst_stats[2] = nb_burst;
1747                         pktnb_stats[2] = nb_pkt;
1748                 }
1749         }
1750         if (total_burst == 0)
1751                 return;
1752
1753         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1754         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1755                 if (i == 3) {
1756                         printf("%d%% of other]\n", 100 - sburstp);
1757                         return;
1758                 }
1759
1760                 sburst += burst_stats[i];
1761                 if (sburst == total_burst) {
1762                         printf("%d%% of %d pkts]\n",
1763                                 100 - sburstp, (int) pktnb_stats[i]);
1764                         return;
1765                 }
1766
1767                 burst_percent[i] =
1768                         (double)burst_stats[i] / total_burst * 100;
1769                 printf("%d%% of %d pkts + ",
1770                         burst_percent[i], (int) pktnb_stats[i]);
1771                 sburstp += burst_percent[i];
1772         }
1773 }
1774
1775 static void
1776 fwd_stream_stats_display(streamid_t stream_id)
1777 {
1778         struct fwd_stream *fs;
1779         static const char *fwd_top_stats_border = "-------";
1780
1781         fs = fwd_streams[stream_id];
1782         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1783             (fs->fwd_dropped == 0))
1784                 return;
1785         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1786                "TX Port=%2d/Queue=%2d %s\n",
1787                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1788                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1789         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1790                " TX-dropped: %-14"PRIu64,
1791                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1792
1793         /* if checksum mode */
1794         if (cur_fwd_eng == &csum_fwd_engine) {
1795                 printf("  RX- bad IP checksum: %-14"PRIu64
1796                        "  Rx- bad L4 checksum: %-14"PRIu64
1797                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1798                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1799                         fs->rx_bad_outer_l4_csum);
1800                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1801                         fs->rx_bad_outer_ip_csum);
1802         } else {
1803                 printf("\n");
1804         }
1805
1806         if (record_burst_stats) {
1807                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1808                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1809         }
1810 }
1811
1812 void
1813 fwd_stats_display(void)
1814 {
1815         static const char *fwd_stats_border = "----------------------";
1816         static const char *acc_stats_border = "+++++++++++++++";
1817         struct {
1818                 struct fwd_stream *rx_stream;
1819                 struct fwd_stream *tx_stream;
1820                 uint64_t tx_dropped;
1821                 uint64_t rx_bad_ip_csum;
1822                 uint64_t rx_bad_l4_csum;
1823                 uint64_t rx_bad_outer_l4_csum;
1824                 uint64_t rx_bad_outer_ip_csum;
1825         } ports_stats[RTE_MAX_ETHPORTS];
1826         uint64_t total_rx_dropped = 0;
1827         uint64_t total_tx_dropped = 0;
1828         uint64_t total_rx_nombuf = 0;
1829         struct rte_eth_stats stats;
1830         uint64_t fwd_cycles = 0;
1831         uint64_t total_recv = 0;
1832         uint64_t total_xmit = 0;
1833         struct rte_port *port;
1834         streamid_t sm_id;
1835         portid_t pt_id;
1836         int i;
1837
1838         memset(ports_stats, 0, sizeof(ports_stats));
1839
1840         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1841                 struct fwd_stream *fs = fwd_streams[sm_id];
1842
1843                 if (cur_fwd_config.nb_fwd_streams >
1844                     cur_fwd_config.nb_fwd_ports) {
1845                         fwd_stream_stats_display(sm_id);
1846                 } else {
1847                         ports_stats[fs->tx_port].tx_stream = fs;
1848                         ports_stats[fs->rx_port].rx_stream = fs;
1849                 }
1850
1851                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1852
1853                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1854                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1855                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1856                                 fs->rx_bad_outer_l4_csum;
1857                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
1858                                 fs->rx_bad_outer_ip_csum;
1859
1860                 if (record_core_cycles)
1861                         fwd_cycles += fs->core_cycles;
1862         }
1863         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1864                 pt_id = fwd_ports_ids[i];
1865                 port = &ports[pt_id];
1866
1867                 rte_eth_stats_get(pt_id, &stats);
1868                 stats.ipackets -= port->stats.ipackets;
1869                 stats.opackets -= port->stats.opackets;
1870                 stats.ibytes -= port->stats.ibytes;
1871                 stats.obytes -= port->stats.obytes;
1872                 stats.imissed -= port->stats.imissed;
1873                 stats.oerrors -= port->stats.oerrors;
1874                 stats.rx_nombuf -= port->stats.rx_nombuf;
1875
1876                 total_recv += stats.ipackets;
1877                 total_xmit += stats.opackets;
1878                 total_rx_dropped += stats.imissed;
1879                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1880                 total_tx_dropped += stats.oerrors;
1881                 total_rx_nombuf  += stats.rx_nombuf;
1882
1883                 printf("\n  %s Forward statistics for port %-2d %s\n",
1884                        fwd_stats_border, pt_id, fwd_stats_border);
1885
1886                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1887                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1888                        stats.ipackets + stats.imissed);
1889
1890                 if (cur_fwd_eng == &csum_fwd_engine) {
1891                         printf("  Bad-ipcsum: %-14"PRIu64
1892                                " Bad-l4csum: %-14"PRIu64
1893                                "Bad-outer-l4csum: %-14"PRIu64"\n",
1894                                ports_stats[pt_id].rx_bad_ip_csum,
1895                                ports_stats[pt_id].rx_bad_l4_csum,
1896                                ports_stats[pt_id].rx_bad_outer_l4_csum);
1897                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
1898                                ports_stats[pt_id].rx_bad_outer_ip_csum);
1899                 }
1900                 if (stats.ierrors + stats.rx_nombuf > 0) {
1901                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
1902                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1903                 }
1904
1905                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1906                        "TX-total: %-"PRIu64"\n",
1907                        stats.opackets, ports_stats[pt_id].tx_dropped,
1908                        stats.opackets + ports_stats[pt_id].tx_dropped);
1909
1910                 if (record_burst_stats) {
1911                         if (ports_stats[pt_id].rx_stream)
1912                                 pkt_burst_stats_display("RX",
1913                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1914                         if (ports_stats[pt_id].tx_stream)
1915                                 pkt_burst_stats_display("TX",
1916                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1917                 }
1918
1919                 printf("  %s--------------------------------%s\n",
1920                        fwd_stats_border, fwd_stats_border);
1921         }
1922
1923         printf("\n  %s Accumulated forward statistics for all ports"
1924                "%s\n",
1925                acc_stats_border, acc_stats_border);
1926         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1927                "%-"PRIu64"\n"
1928                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1929                "%-"PRIu64"\n",
1930                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1931                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1932         if (total_rx_nombuf > 0)
1933                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1934         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1935                "%s\n",
1936                acc_stats_border, acc_stats_border);
1937         if (record_core_cycles) {
1938 #define CYC_PER_MHZ 1E6
1939                 if (total_recv > 0 || total_xmit > 0) {
1940                         uint64_t total_pkts = 0;
1941                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1942                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1943                                 total_pkts = total_xmit;
1944                         else
1945                                 total_pkts = total_recv;
1946
1947                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1948                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1949                                " MHz Clock\n",
1950                                (double) fwd_cycles / total_pkts,
1951                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1952                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1953                 }
1954         }
1955 }
1956
1957 void
1958 fwd_stats_reset(void)
1959 {
1960         streamid_t sm_id;
1961         portid_t pt_id;
1962         int i;
1963
1964         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1965                 pt_id = fwd_ports_ids[i];
1966                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1967         }
1968         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1969                 struct fwd_stream *fs = fwd_streams[sm_id];
1970
1971                 fs->rx_packets = 0;
1972                 fs->tx_packets = 0;
1973                 fs->fwd_dropped = 0;
1974                 fs->rx_bad_ip_csum = 0;
1975                 fs->rx_bad_l4_csum = 0;
1976                 fs->rx_bad_outer_l4_csum = 0;
1977                 fs->rx_bad_outer_ip_csum = 0;
1978
1979                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1980                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1981                 fs->core_cycles = 0;
1982         }
1983 }
1984
1985 static void
1986 flush_fwd_rx_queues(void)
1987 {
1988         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1989         portid_t  rxp;
1990         portid_t port_id;
1991         queueid_t rxq;
1992         uint16_t  nb_rx;
1993         uint16_t  i;
1994         uint8_t   j;
1995         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1996         uint64_t timer_period;
1997
1998         /* convert to number of cycles */
1999         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2000
2001         for (j = 0; j < 2; j++) {
2002                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2003                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2004                                 port_id = fwd_ports_ids[rxp];
2005                                 /**
2006                                 * testpmd can stuck in the below do while loop
2007                                 * if rte_eth_rx_burst() always returns nonzero
2008                                 * packets. So timer is added to exit this loop
2009                                 * after 1sec timer expiry.
2010                                 */
2011                                 prev_tsc = rte_rdtsc();
2012                                 do {
2013                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2014                                                 pkts_burst, MAX_PKT_BURST);
2015                                         for (i = 0; i < nb_rx; i++)
2016                                                 rte_pktmbuf_free(pkts_burst[i]);
2017
2018                                         cur_tsc = rte_rdtsc();
2019                                         diff_tsc = cur_tsc - prev_tsc;
2020                                         timer_tsc += diff_tsc;
2021                                 } while ((nb_rx > 0) &&
2022                                         (timer_tsc < timer_period));
2023                                 timer_tsc = 0;
2024                         }
2025                 }
2026                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2027         }
2028 }
2029
2030 static void
2031 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2032 {
2033         struct fwd_stream **fsm;
2034         streamid_t nb_fs;
2035         streamid_t sm_id;
2036 #ifdef RTE_LIB_BITRATESTATS
2037         uint64_t tics_per_1sec;
2038         uint64_t tics_datum;
2039         uint64_t tics_current;
2040         uint16_t i, cnt_ports;
2041
2042         cnt_ports = nb_ports;
2043         tics_datum = rte_rdtsc();
2044         tics_per_1sec = rte_get_timer_hz();
2045 #endif
2046         fsm = &fwd_streams[fc->stream_idx];
2047         nb_fs = fc->stream_nb;
2048         do {
2049                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2050                         (*pkt_fwd)(fsm[sm_id]);
2051 #ifdef RTE_LIB_BITRATESTATS
2052                 if (bitrate_enabled != 0 &&
2053                                 bitrate_lcore_id == rte_lcore_id()) {
2054                         tics_current = rte_rdtsc();
2055                         if (tics_current - tics_datum >= tics_per_1sec) {
2056                                 /* Periodic bitrate calculation */
2057                                 for (i = 0; i < cnt_ports; i++)
2058                                         rte_stats_bitrate_calc(bitrate_data,
2059                                                 ports_ids[i]);
2060                                 tics_datum = tics_current;
2061                         }
2062                 }
2063 #endif
2064 #ifdef RTE_LIB_LATENCYSTATS
2065                 if (latencystats_enabled != 0 &&
2066                                 latencystats_lcore_id == rte_lcore_id())
2067                         rte_latencystats_update();
2068 #endif
2069
2070         } while (! fc->stopped);
2071 }
2072
2073 static int
2074 start_pkt_forward_on_core(void *fwd_arg)
2075 {
2076         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2077                              cur_fwd_config.fwd_eng->packet_fwd);
2078         return 0;
2079 }
2080
2081 /*
2082  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2083  * Used to start communication flows in network loopback test configurations.
2084  */
2085 static int
2086 run_one_txonly_burst_on_core(void *fwd_arg)
2087 {
2088         struct fwd_lcore *fwd_lc;
2089         struct fwd_lcore tmp_lcore;
2090
2091         fwd_lc = (struct fwd_lcore *) fwd_arg;
2092         tmp_lcore = *fwd_lc;
2093         tmp_lcore.stopped = 1;
2094         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2095         return 0;
2096 }
2097
2098 /*
2099  * Launch packet forwarding:
2100  *     - Setup per-port forwarding context.
2101  *     - launch logical cores with their forwarding configuration.
2102  */
2103 static void
2104 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2105 {
2106         port_fwd_begin_t port_fwd_begin;
2107         unsigned int i;
2108         unsigned int lc_id;
2109         int diag;
2110
2111         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2112         if (port_fwd_begin != NULL) {
2113                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2114                         (*port_fwd_begin)(fwd_ports_ids[i]);
2115         }
2116         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2117                 lc_id = fwd_lcores_cpuids[i];
2118                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2119                         fwd_lcores[i]->stopped = 0;
2120                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2121                                                      fwd_lcores[i], lc_id);
2122                         if (diag != 0)
2123                                 fprintf(stderr,
2124                                         "launch lcore %u failed - diag=%d\n",
2125                                         lc_id, diag);
2126                 }
2127         }
2128 }
2129
2130 /*
2131  * Launch packet forwarding configuration.
2132  */
2133 void
2134 start_packet_forwarding(int with_tx_first)
2135 {
2136         port_fwd_begin_t port_fwd_begin;
2137         port_fwd_end_t  port_fwd_end;
2138         unsigned int i;
2139
2140         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2141                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2142
2143         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2144                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2145
2146         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2147                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2148                 (!nb_rxq || !nb_txq))
2149                 rte_exit(EXIT_FAILURE,
2150                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2151                         cur_fwd_eng->fwd_mode_name);
2152
2153         if (all_ports_started() == 0) {
2154                 fprintf(stderr, "Not all ports were started\n");
2155                 return;
2156         }
2157         if (test_done == 0) {
2158                 fprintf(stderr, "Packet forwarding already started\n");
2159                 return;
2160         }
2161         test_done = 0;
2162
2163         fwd_config_setup();
2164
2165         if(!no_flush_rx)
2166                 flush_fwd_rx_queues();
2167
2168         pkt_fwd_config_display(&cur_fwd_config);
2169         rxtx_config_display();
2170
2171         fwd_stats_reset();
2172         if (with_tx_first) {
2173                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2174                 if (port_fwd_begin != NULL) {
2175                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2176                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2177                 }
2178                 while (with_tx_first--) {
2179                         launch_packet_forwarding(
2180                                         run_one_txonly_burst_on_core);
2181                         rte_eal_mp_wait_lcore();
2182                 }
2183                 port_fwd_end = tx_only_engine.port_fwd_end;
2184                 if (port_fwd_end != NULL) {
2185                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2186                                 (*port_fwd_end)(fwd_ports_ids[i]);
2187                 }
2188         }
2189         launch_packet_forwarding(start_pkt_forward_on_core);
2190 }
2191
2192 void
2193 stop_packet_forwarding(void)
2194 {
2195         port_fwd_end_t port_fwd_end;
2196         lcoreid_t lc_id;
2197         portid_t pt_id;
2198         int i;
2199
2200         if (test_done) {
2201                 fprintf(stderr, "Packet forwarding not started\n");
2202                 return;
2203         }
2204         printf("Telling cores to stop...");
2205         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2206                 fwd_lcores[lc_id]->stopped = 1;
2207         printf("\nWaiting for lcores to finish...\n");
2208         rte_eal_mp_wait_lcore();
2209         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2210         if (port_fwd_end != NULL) {
2211                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2212                         pt_id = fwd_ports_ids[i];
2213                         (*port_fwd_end)(pt_id);
2214                 }
2215         }
2216
2217         fwd_stats_display();
2218
2219         printf("\nDone.\n");
2220         test_done = 1;
2221 }
2222
2223 void
2224 dev_set_link_up(portid_t pid)
2225 {
2226         if (rte_eth_dev_set_link_up(pid) < 0)
2227                 fprintf(stderr, "\nSet link up fail.\n");
2228 }
2229
2230 void
2231 dev_set_link_down(portid_t pid)
2232 {
2233         if (rte_eth_dev_set_link_down(pid) < 0)
2234                 fprintf(stderr, "\nSet link down fail.\n");
2235 }
2236
2237 static int
2238 all_ports_started(void)
2239 {
2240         portid_t pi;
2241         struct rte_port *port;
2242
2243         RTE_ETH_FOREACH_DEV(pi) {
2244                 port = &ports[pi];
2245                 /* Check if there is a port which is not started */
2246                 if ((port->port_status != RTE_PORT_STARTED) &&
2247                         (port->slave_flag == 0))
2248                         return 0;
2249         }
2250
2251         /* No port is not started */
2252         return 1;
2253 }
2254
2255 int
2256 port_is_stopped(portid_t port_id)
2257 {
2258         struct rte_port *port = &ports[port_id];
2259
2260         if ((port->port_status != RTE_PORT_STOPPED) &&
2261             (port->slave_flag == 0))
2262                 return 0;
2263         return 1;
2264 }
2265
2266 int
2267 all_ports_stopped(void)
2268 {
2269         portid_t pi;
2270
2271         RTE_ETH_FOREACH_DEV(pi) {
2272                 if (!port_is_stopped(pi))
2273                         return 0;
2274         }
2275
2276         return 1;
2277 }
2278
2279 int
2280 port_is_started(portid_t port_id)
2281 {
2282         if (port_id_is_invalid(port_id, ENABLED_WARN))
2283                 return 0;
2284
2285         if (ports[port_id].port_status != RTE_PORT_STARTED)
2286                 return 0;
2287
2288         return 1;
2289 }
2290
2291 /* Configure the Rx and Tx hairpin queues for the selected port. */
2292 static int
2293 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2294 {
2295         queueid_t qi;
2296         struct rte_eth_hairpin_conf hairpin_conf = {
2297                 .peer_count = 1,
2298         };
2299         int i;
2300         int diag;
2301         struct rte_port *port = &ports[pi];
2302         uint16_t peer_rx_port = pi;
2303         uint16_t peer_tx_port = pi;
2304         uint32_t manual = 1;
2305         uint32_t tx_exp = hairpin_mode & 0x10;
2306
2307         if (!(hairpin_mode & 0xf)) {
2308                 peer_rx_port = pi;
2309                 peer_tx_port = pi;
2310                 manual = 0;
2311         } else if (hairpin_mode & 0x1) {
2312                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2313                                                        RTE_ETH_DEV_NO_OWNER);
2314                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2315                         peer_tx_port = rte_eth_find_next_owned_by(0,
2316                                                 RTE_ETH_DEV_NO_OWNER);
2317                 if (p_pi != RTE_MAX_ETHPORTS) {
2318                         peer_rx_port = p_pi;
2319                 } else {
2320                         uint16_t next_pi;
2321
2322                         /* Last port will be the peer RX port of the first. */
2323                         RTE_ETH_FOREACH_DEV(next_pi)
2324                                 peer_rx_port = next_pi;
2325                 }
2326                 manual = 1;
2327         } else if (hairpin_mode & 0x2) {
2328                 if (cnt_pi & 0x1) {
2329                         peer_rx_port = p_pi;
2330                 } else {
2331                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2332                                                 RTE_ETH_DEV_NO_OWNER);
2333                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2334                                 peer_rx_port = pi;
2335                 }
2336                 peer_tx_port = peer_rx_port;
2337                 manual = 1;
2338         }
2339
2340         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2341                 hairpin_conf.peers[0].port = peer_rx_port;
2342                 hairpin_conf.peers[0].queue = i + nb_rxq;
2343                 hairpin_conf.manual_bind = !!manual;
2344                 hairpin_conf.tx_explicit = !!tx_exp;
2345                 diag = rte_eth_tx_hairpin_queue_setup
2346                         (pi, qi, nb_txd, &hairpin_conf);
2347                 i++;
2348                 if (diag == 0)
2349                         continue;
2350
2351                 /* Fail to setup rx queue, return */
2352                 if (rte_atomic16_cmpset(&(port->port_status),
2353                                         RTE_PORT_HANDLING,
2354                                         RTE_PORT_STOPPED) == 0)
2355                         fprintf(stderr,
2356                                 "Port %d can not be set back to stopped\n", pi);
2357                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2358                         pi);
2359                 /* try to reconfigure queues next time */
2360                 port->need_reconfig_queues = 1;
2361                 return -1;
2362         }
2363         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2364                 hairpin_conf.peers[0].port = peer_tx_port;
2365                 hairpin_conf.peers[0].queue = i + nb_txq;
2366                 hairpin_conf.manual_bind = !!manual;
2367                 hairpin_conf.tx_explicit = !!tx_exp;
2368                 diag = rte_eth_rx_hairpin_queue_setup
2369                         (pi, qi, nb_rxd, &hairpin_conf);
2370                 i++;
2371                 if (diag == 0)
2372                         continue;
2373
2374                 /* Fail to setup rx queue, return */
2375                 if (rte_atomic16_cmpset(&(port->port_status),
2376                                         RTE_PORT_HANDLING,
2377                                         RTE_PORT_STOPPED) == 0)
2378                         fprintf(stderr,
2379                                 "Port %d can not be set back to stopped\n", pi);
2380                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2381                         pi);
2382                 /* try to reconfigure queues next time */
2383                 port->need_reconfig_queues = 1;
2384                 return -1;
2385         }
2386         return 0;
2387 }
2388
2389 /* Configure the Rx with optional split. */
2390 int
2391 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2392                uint16_t nb_rx_desc, unsigned int socket_id,
2393                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2394 {
2395         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2396         unsigned int i, mp_n;
2397         int ret;
2398
2399         if (rx_pkt_nb_segs <= 1 ||
2400             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2401                 rx_conf->rx_seg = NULL;
2402                 rx_conf->rx_nseg = 0;
2403                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2404                                              nb_rx_desc, socket_id,
2405                                              rx_conf, mp);
2406                 return ret;
2407         }
2408         for (i = 0; i < rx_pkt_nb_segs; i++) {
2409                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2410                 struct rte_mempool *mpx;
2411                 /*
2412                  * Use last valid pool for the segments with number
2413                  * exceeding the pool index.
2414                  */
2415                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2416                 mpx = mbuf_pool_find(socket_id, mp_n);
2417                 /* Handle zero as mbuf data buffer size. */
2418                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2419                                    rx_pkt_seg_lengths[i] :
2420                                    mbuf_data_size[mp_n];
2421                 rx_seg->offset = i < rx_pkt_nb_offs ?
2422                                    rx_pkt_seg_offsets[i] : 0;
2423                 rx_seg->mp = mpx ? mpx : mp;
2424         }
2425         rx_conf->rx_nseg = rx_pkt_nb_segs;
2426         rx_conf->rx_seg = rx_useg;
2427         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2428                                     socket_id, rx_conf, NULL);
2429         rx_conf->rx_seg = NULL;
2430         rx_conf->rx_nseg = 0;
2431         return ret;
2432 }
2433
2434 int
2435 start_port(portid_t pid)
2436 {
2437         int diag, need_check_link_status = -1;
2438         portid_t pi;
2439         portid_t p_pi = RTE_MAX_ETHPORTS;
2440         portid_t pl[RTE_MAX_ETHPORTS];
2441         portid_t peer_pl[RTE_MAX_ETHPORTS];
2442         uint16_t cnt_pi = 0;
2443         uint16_t cfg_pi = 0;
2444         int peer_pi;
2445         queueid_t qi;
2446         struct rte_port *port;
2447         struct rte_eth_hairpin_cap cap;
2448
2449         if (port_id_is_invalid(pid, ENABLED_WARN))
2450                 return 0;
2451
2452         RTE_ETH_FOREACH_DEV(pi) {
2453                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2454                         continue;
2455
2456                 need_check_link_status = 0;
2457                 port = &ports[pi];
2458                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2459                                                  RTE_PORT_HANDLING) == 0) {
2460                         fprintf(stderr, "Port %d is now not stopped\n", pi);
2461                         continue;
2462                 }
2463
2464                 if (port->need_reconfig > 0) {
2465                         port->need_reconfig = 0;
2466
2467                         if (flow_isolate_all) {
2468                                 int ret = port_flow_isolate(pi, 1);
2469                                 if (ret) {
2470                                         fprintf(stderr,
2471                                                 "Failed to apply isolated mode on port %d\n",
2472                                                 pi);
2473                                         return -1;
2474                                 }
2475                         }
2476                         configure_rxtx_dump_callbacks(0);
2477                         printf("Configuring Port %d (socket %u)\n", pi,
2478                                         port->socket_id);
2479                         if (nb_hairpinq > 0 &&
2480                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2481                                 fprintf(stderr,
2482                                         "Port %d doesn't support hairpin queues\n",
2483                                         pi);
2484                                 return -1;
2485                         }
2486                         /* configure port */
2487                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2488                                                      nb_txq + nb_hairpinq,
2489                                                      &(port->dev_conf));
2490                         if (diag != 0) {
2491                                 if (rte_atomic16_cmpset(&(port->port_status),
2492                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2493                                         fprintf(stderr,
2494                                                 "Port %d can not be set back to stopped\n",
2495                                                 pi);
2496                                 fprintf(stderr, "Fail to configure port %d\n",
2497                                         pi);
2498                                 /* try to reconfigure port next time */
2499                                 port->need_reconfig = 1;
2500                                 return -1;
2501                         }
2502                 }
2503                 if (port->need_reconfig_queues > 0) {
2504                         port->need_reconfig_queues = 0;
2505                         /* setup tx queues */
2506                         for (qi = 0; qi < nb_txq; qi++) {
2507                                 if ((numa_support) &&
2508                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2509                                         diag = rte_eth_tx_queue_setup(pi, qi,
2510                                                 port->nb_tx_desc[qi],
2511                                                 txring_numa[pi],
2512                                                 &(port->tx_conf[qi]));
2513                                 else
2514                                         diag = rte_eth_tx_queue_setup(pi, qi,
2515                                                 port->nb_tx_desc[qi],
2516                                                 port->socket_id,
2517                                                 &(port->tx_conf[qi]));
2518
2519                                 if (diag == 0)
2520                                         continue;
2521
2522                                 /* Fail to setup tx queue, return */
2523                                 if (rte_atomic16_cmpset(&(port->port_status),
2524                                                         RTE_PORT_HANDLING,
2525                                                         RTE_PORT_STOPPED) == 0)
2526                                         fprintf(stderr,
2527                                                 "Port %d can not be set back to stopped\n",
2528                                                 pi);
2529                                 fprintf(stderr,
2530                                         "Fail to configure port %d tx queues\n",
2531                                         pi);
2532                                 /* try to reconfigure queues next time */
2533                                 port->need_reconfig_queues = 1;
2534                                 return -1;
2535                         }
2536                         for (qi = 0; qi < nb_rxq; qi++) {
2537                                 /* setup rx queues */
2538                                 if ((numa_support) &&
2539                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2540                                         struct rte_mempool * mp =
2541                                                 mbuf_pool_find
2542                                                         (rxring_numa[pi], 0);
2543                                         if (mp == NULL) {
2544                                                 fprintf(stderr,
2545                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2546                                                         rxring_numa[pi]);
2547                                                 return -1;
2548                                         }
2549
2550                                         diag = rx_queue_setup(pi, qi,
2551                                              port->nb_rx_desc[qi],
2552                                              rxring_numa[pi],
2553                                              &(port->rx_conf[qi]),
2554                                              mp);
2555                                 } else {
2556                                         struct rte_mempool *mp =
2557                                                 mbuf_pool_find
2558                                                         (port->socket_id, 0);
2559                                         if (mp == NULL) {
2560                                                 fprintf(stderr,
2561                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2562                                                         port->socket_id);
2563                                                 return -1;
2564                                         }
2565                                         diag = rx_queue_setup(pi, qi,
2566                                              port->nb_rx_desc[qi],
2567                                              port->socket_id,
2568                                              &(port->rx_conf[qi]),
2569                                              mp);
2570                                 }
2571                                 if (diag == 0)
2572                                         continue;
2573
2574                                 /* Fail to setup rx queue, return */
2575                                 if (rte_atomic16_cmpset(&(port->port_status),
2576                                                         RTE_PORT_HANDLING,
2577                                                         RTE_PORT_STOPPED) == 0)
2578                                         fprintf(stderr,
2579                                                 "Port %d can not be set back to stopped\n",
2580                                                 pi);
2581                                 fprintf(stderr,
2582                                         "Fail to configure port %d rx queues\n",
2583                                         pi);
2584                                 /* try to reconfigure queues next time */
2585                                 port->need_reconfig_queues = 1;
2586                                 return -1;
2587                         }
2588                         /* setup hairpin queues */
2589                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2590                                 return -1;
2591                 }
2592                 configure_rxtx_dump_callbacks(verbose_level);
2593                 if (clear_ptypes) {
2594                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2595                                         NULL, 0);
2596                         if (diag < 0)
2597                                 fprintf(stderr,
2598                                         "Port %d: Failed to disable Ptype parsing\n",
2599                                         pi);
2600                 }
2601
2602                 p_pi = pi;
2603                 cnt_pi++;
2604
2605                 /* start port */
2606                 diag = rte_eth_dev_start(pi);
2607                 if (diag < 0) {
2608                         fprintf(stderr, "Fail to start port %d: %s\n",
2609                                 pi, rte_strerror(-diag));
2610
2611                         /* Fail to setup rx queue, return */
2612                         if (rte_atomic16_cmpset(&(port->port_status),
2613                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2614                                 fprintf(stderr,
2615                                         "Port %d can not be set back to stopped\n",
2616                                         pi);
2617                         continue;
2618                 }
2619
2620                 if (rte_atomic16_cmpset(&(port->port_status),
2621                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2622                         fprintf(stderr, "Port %d can not be set into started\n",
2623                                 pi);
2624
2625                 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
2626                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2627                                 port->eth_addr.addr_bytes[0],
2628                                 port->eth_addr.addr_bytes[1],
2629                                 port->eth_addr.addr_bytes[2],
2630                                 port->eth_addr.addr_bytes[3],
2631                                 port->eth_addr.addr_bytes[4],
2632                                 port->eth_addr.addr_bytes[5]);
2633
2634                 /* at least one port started, need checking link status */
2635                 need_check_link_status = 1;
2636
2637                 pl[cfg_pi++] = pi;
2638         }
2639
2640         if (need_check_link_status == 1 && !no_link_check)
2641                 check_all_ports_link_status(RTE_PORT_ALL);
2642         else if (need_check_link_status == 0)
2643                 fprintf(stderr, "Please stop the ports first\n");
2644
2645         if (hairpin_mode & 0xf) {
2646                 uint16_t i;
2647                 int j;
2648
2649                 /* bind all started hairpin ports */
2650                 for (i = 0; i < cfg_pi; i++) {
2651                         pi = pl[i];
2652                         /* bind current Tx to all peer Rx */
2653                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2654                                                         RTE_MAX_ETHPORTS, 1);
2655                         if (peer_pi < 0)
2656                                 return peer_pi;
2657                         for (j = 0; j < peer_pi; j++) {
2658                                 if (!port_is_started(peer_pl[j]))
2659                                         continue;
2660                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2661                                 if (diag < 0) {
2662                                         fprintf(stderr,
2663                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
2664                                                 pi, peer_pl[j],
2665                                                 rte_strerror(-diag));
2666                                         return -1;
2667                                 }
2668                         }
2669                         /* bind all peer Tx to current Rx */
2670                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2671                                                         RTE_MAX_ETHPORTS, 0);
2672                         if (peer_pi < 0)
2673                                 return peer_pi;
2674                         for (j = 0; j < peer_pi; j++) {
2675                                 if (!port_is_started(peer_pl[j]))
2676                                         continue;
2677                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2678                                 if (diag < 0) {
2679                                         fprintf(stderr,
2680                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
2681                                                 peer_pl[j], pi,
2682                                                 rte_strerror(-diag));
2683                                         return -1;
2684                                 }
2685                         }
2686                 }
2687         }
2688
2689         printf("Done\n");
2690         return 0;
2691 }
2692
2693 void
2694 stop_port(portid_t pid)
2695 {
2696         portid_t pi;
2697         struct rte_port *port;
2698         int need_check_link_status = 0;
2699         portid_t peer_pl[RTE_MAX_ETHPORTS];
2700         int peer_pi;
2701
2702         if (port_id_is_invalid(pid, ENABLED_WARN))
2703                 return;
2704
2705         printf("Stopping ports...\n");
2706
2707         RTE_ETH_FOREACH_DEV(pi) {
2708                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2709                         continue;
2710
2711                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2712                         fprintf(stderr,
2713                                 "Please remove port %d from forwarding configuration.\n",
2714                                 pi);
2715                         continue;
2716                 }
2717
2718                 if (port_is_bonding_slave(pi)) {
2719                         fprintf(stderr,
2720                                 "Please remove port %d from bonded device.\n",
2721                                 pi);
2722                         continue;
2723                 }
2724
2725                 port = &ports[pi];
2726                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2727                                                 RTE_PORT_HANDLING) == 0)
2728                         continue;
2729
2730                 if (hairpin_mode & 0xf) {
2731                         int j;
2732
2733                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2734                         /* unbind all peer Tx from current Rx */
2735                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2736                                                         RTE_MAX_ETHPORTS, 0);
2737                         if (peer_pi < 0)
2738                                 continue;
2739                         for (j = 0; j < peer_pi; j++) {
2740                                 if (!port_is_started(peer_pl[j]))
2741                                         continue;
2742                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2743                         }
2744                 }
2745
2746                 if (port->flow_list)
2747                         port_flow_flush(pi);
2748
2749                 if (rte_eth_dev_stop(pi) != 0)
2750                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2751                                 pi);
2752
2753                 if (rte_atomic16_cmpset(&(port->port_status),
2754                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2755                         fprintf(stderr, "Port %d can not be set into stopped\n",
2756                                 pi);
2757                 need_check_link_status = 1;
2758         }
2759         if (need_check_link_status && !no_link_check)
2760                 check_all_ports_link_status(RTE_PORT_ALL);
2761
2762         printf("Done\n");
2763 }
2764
2765 static void
2766 remove_invalid_ports_in(portid_t *array, portid_t *total)
2767 {
2768         portid_t i;
2769         portid_t new_total = 0;
2770
2771         for (i = 0; i < *total; i++)
2772                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2773                         array[new_total] = array[i];
2774                         new_total++;
2775                 }
2776         *total = new_total;
2777 }
2778
2779 static void
2780 remove_invalid_ports(void)
2781 {
2782         remove_invalid_ports_in(ports_ids, &nb_ports);
2783         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2784         nb_cfg_ports = nb_fwd_ports;
2785 }
2786
2787 void
2788 close_port(portid_t pid)
2789 {
2790         portid_t pi;
2791         struct rte_port *port;
2792
2793         if (port_id_is_invalid(pid, ENABLED_WARN))
2794                 return;
2795
2796         printf("Closing ports...\n");
2797
2798         RTE_ETH_FOREACH_DEV(pi) {
2799                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2800                         continue;
2801
2802                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2803                         fprintf(stderr,
2804                                 "Please remove port %d from forwarding configuration.\n",
2805                                 pi);
2806                         continue;
2807                 }
2808
2809                 if (port_is_bonding_slave(pi)) {
2810                         fprintf(stderr,
2811                                 "Please remove port %d from bonded device.\n",
2812                                 pi);
2813                         continue;
2814                 }
2815
2816                 port = &ports[pi];
2817                 if (rte_atomic16_cmpset(&(port->port_status),
2818                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2819                         fprintf(stderr, "Port %d is already closed\n", pi);
2820                         continue;
2821                 }
2822
2823                 port_flow_flush(pi);
2824                 rte_eth_dev_close(pi);
2825         }
2826
2827         remove_invalid_ports();
2828         printf("Done\n");
2829 }
2830
2831 void
2832 reset_port(portid_t pid)
2833 {
2834         int diag;
2835         portid_t pi;
2836         struct rte_port *port;
2837
2838         if (port_id_is_invalid(pid, ENABLED_WARN))
2839                 return;
2840
2841         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2842                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2843                 fprintf(stderr,
2844                         "Can not reset port(s), please stop port(s) first.\n");
2845                 return;
2846         }
2847
2848         printf("Resetting ports...\n");
2849
2850         RTE_ETH_FOREACH_DEV(pi) {
2851                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2852                         continue;
2853
2854                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2855                         fprintf(stderr,
2856                                 "Please remove port %d from forwarding configuration.\n",
2857                                 pi);
2858                         continue;
2859                 }
2860
2861                 if (port_is_bonding_slave(pi)) {
2862                         fprintf(stderr,
2863                                 "Please remove port %d from bonded device.\n",
2864                                 pi);
2865                         continue;
2866                 }
2867
2868                 diag = rte_eth_dev_reset(pi);
2869                 if (diag == 0) {
2870                         port = &ports[pi];
2871                         port->need_reconfig = 1;
2872                         port->need_reconfig_queues = 1;
2873                 } else {
2874                         fprintf(stderr, "Failed to reset port %d. diag=%d\n",
2875                                 pi, diag);
2876                 }
2877         }
2878
2879         printf("Done\n");
2880 }
2881
2882 void
2883 attach_port(char *identifier)
2884 {
2885         portid_t pi;
2886         struct rte_dev_iterator iterator;
2887
2888         printf("Attaching a new port...\n");
2889
2890         if (identifier == NULL) {
2891                 fprintf(stderr, "Invalid parameters are specified\n");
2892                 return;
2893         }
2894
2895         if (rte_dev_probe(identifier) < 0) {
2896                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2897                 return;
2898         }
2899
2900         /* first attach mode: event */
2901         if (setup_on_probe_event) {
2902                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2903                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2904                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2905                                         ports[pi].need_setup != 0)
2906                                 setup_attached_port(pi);
2907                 return;
2908         }
2909
2910         /* second attach mode: iterator */
2911         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2912                 /* setup ports matching the devargs used for probing */
2913                 if (port_is_forwarding(pi))
2914                         continue; /* port was already attached before */
2915                 setup_attached_port(pi);
2916         }
2917 }
2918
2919 static void
2920 setup_attached_port(portid_t pi)
2921 {
2922         unsigned int socket_id;
2923         int ret;
2924
2925         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2926         /* if socket_id is invalid, set to the first available socket. */
2927         if (check_socket_id(socket_id) < 0)
2928                 socket_id = socket_ids[0];
2929         reconfig(pi, socket_id);
2930         ret = rte_eth_promiscuous_enable(pi);
2931         if (ret != 0)
2932                 fprintf(stderr,
2933                         "Error during enabling promiscuous mode for port %u: %s - ignore\n",
2934                         pi, rte_strerror(-ret));
2935
2936         ports_ids[nb_ports++] = pi;
2937         fwd_ports_ids[nb_fwd_ports++] = pi;
2938         nb_cfg_ports = nb_fwd_ports;
2939         ports[pi].need_setup = 0;
2940         ports[pi].port_status = RTE_PORT_STOPPED;
2941
2942         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2943         printf("Done\n");
2944 }
2945
2946 static void
2947 detach_device(struct rte_device *dev)
2948 {
2949         portid_t sibling;
2950
2951         if (dev == NULL) {
2952                 fprintf(stderr, "Device already removed\n");
2953                 return;
2954         }
2955
2956         printf("Removing a device...\n");
2957
2958         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2959                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2960                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2961                                 fprintf(stderr, "Port %u not stopped\n",
2962                                         sibling);
2963                                 return;
2964                         }
2965                         port_flow_flush(sibling);
2966                 }
2967         }
2968
2969         if (rte_dev_remove(dev) < 0) {
2970                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2971                 return;
2972         }
2973         remove_invalid_ports();
2974
2975         printf("Device is detached\n");
2976         printf("Now total ports is %d\n", nb_ports);
2977         printf("Done\n");
2978         return;
2979 }
2980
2981 void
2982 detach_port_device(portid_t port_id)
2983 {
2984         int ret;
2985         struct rte_eth_dev_info dev_info;
2986
2987         if (port_id_is_invalid(port_id, ENABLED_WARN))
2988                 return;
2989
2990         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2991                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2992                         fprintf(stderr, "Port not stopped\n");
2993                         return;
2994                 }
2995                 fprintf(stderr, "Port was not closed\n");
2996         }
2997
2998         ret = eth_dev_info_get_print_err(port_id, &dev_info);
2999         if (ret != 0) {
3000                 TESTPMD_LOG(ERR,
3001                         "Failed to get device info for port %d, not detaching\n",
3002                         port_id);
3003                 return;
3004         }
3005         detach_device(dev_info.device);
3006 }
3007
3008 void
3009 detach_devargs(char *identifier)
3010 {
3011         struct rte_dev_iterator iterator;
3012         struct rte_devargs da;
3013         portid_t port_id;
3014
3015         printf("Removing a device...\n");
3016
3017         memset(&da, 0, sizeof(da));
3018         if (rte_devargs_parsef(&da, "%s", identifier)) {
3019                 fprintf(stderr, "cannot parse identifier\n");
3020                 return;
3021         }
3022
3023         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3024                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3025                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3026                                 fprintf(stderr, "Port %u not stopped\n",
3027                                         port_id);
3028                                 rte_eth_iterator_cleanup(&iterator);
3029                                 rte_devargs_reset(&da);
3030                                 return;
3031                         }
3032                         port_flow_flush(port_id);
3033                 }
3034         }
3035
3036         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3037                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3038                             da.name, da.bus->name);
3039                 rte_devargs_reset(&da);
3040                 return;
3041         }
3042
3043         remove_invalid_ports();
3044
3045         printf("Device %s is detached\n", identifier);
3046         printf("Now total ports is %d\n", nb_ports);
3047         printf("Done\n");
3048         rte_devargs_reset(&da);
3049 }
3050
3051 void
3052 pmd_test_exit(void)
3053 {
3054         portid_t pt_id;
3055         unsigned int i;
3056         int ret;
3057
3058         if (test_done == 0)
3059                 stop_packet_forwarding();
3060
3061 #ifndef RTE_EXEC_ENV_WINDOWS
3062         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3063                 if (mempools[i]) {
3064                         if (mp_alloc_type == MP_ALLOC_ANON)
3065                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3066                                                      NULL);
3067                 }
3068         }
3069 #endif
3070         if (ports != NULL) {
3071                 no_link_check = 1;
3072                 RTE_ETH_FOREACH_DEV(pt_id) {
3073                         printf("\nStopping port %d...\n", pt_id);
3074                         fflush(stdout);
3075                         stop_port(pt_id);
3076                 }
3077                 RTE_ETH_FOREACH_DEV(pt_id) {
3078                         printf("\nShutting down port %d...\n", pt_id);
3079                         fflush(stdout);
3080                         close_port(pt_id);
3081                 }
3082         }
3083
3084         if (hot_plug) {
3085                 ret = rte_dev_event_monitor_stop();
3086                 if (ret) {
3087                         RTE_LOG(ERR, EAL,
3088                                 "fail to stop device event monitor.");
3089                         return;
3090                 }
3091
3092                 ret = rte_dev_event_callback_unregister(NULL,
3093                         dev_event_callback, NULL);
3094                 if (ret < 0) {
3095                         RTE_LOG(ERR, EAL,
3096                                 "fail to unregister device event callback.\n");
3097                         return;
3098                 }
3099
3100                 ret = rte_dev_hotplug_handle_disable();
3101                 if (ret) {
3102                         RTE_LOG(ERR, EAL,
3103                                 "fail to disable hotplug handling.\n");
3104                         return;
3105                 }
3106         }
3107         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3108                 if (mempools[i])
3109                         rte_mempool_free(mempools[i]);
3110         }
3111
3112         printf("\nBye...\n");
3113 }
3114
3115 typedef void (*cmd_func_t)(void);
3116 struct pmd_test_command {
3117         const char *cmd_name;
3118         cmd_func_t cmd_func;
3119 };
3120
3121 /* Check the link status of all ports in up to 9s, and print them finally */
3122 static void
3123 check_all_ports_link_status(uint32_t port_mask)
3124 {
3125 #define CHECK_INTERVAL 100 /* 100ms */
3126 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3127         portid_t portid;
3128         uint8_t count, all_ports_up, print_flag = 0;
3129         struct rte_eth_link link;
3130         int ret;
3131         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3132
3133         printf("Checking link statuses...\n");
3134         fflush(stdout);
3135         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3136                 all_ports_up = 1;
3137                 RTE_ETH_FOREACH_DEV(portid) {
3138                         if ((port_mask & (1 << portid)) == 0)
3139                                 continue;
3140                         memset(&link, 0, sizeof(link));
3141                         ret = rte_eth_link_get_nowait(portid, &link);
3142                         if (ret < 0) {
3143                                 all_ports_up = 0;
3144                                 if (print_flag == 1)
3145                                         fprintf(stderr,
3146                                                 "Port %u link get failed: %s\n",
3147                                                 portid, rte_strerror(-ret));
3148                                 continue;
3149                         }
3150                         /* print link status if flag set */
3151                         if (print_flag == 1) {
3152                                 rte_eth_link_to_str(link_status,
3153                                         sizeof(link_status), &link);
3154                                 printf("Port %d %s\n", portid, link_status);
3155                                 continue;
3156                         }
3157                         /* clear all_ports_up flag if any link down */
3158                         if (link.link_status == ETH_LINK_DOWN) {
3159                                 all_ports_up = 0;
3160                                 break;
3161                         }
3162                 }
3163                 /* after finally printing all link status, get out */
3164                 if (print_flag == 1)
3165                         break;
3166
3167                 if (all_ports_up == 0) {
3168                         fflush(stdout);
3169                         rte_delay_ms(CHECK_INTERVAL);
3170                 }
3171
3172                 /* set the print_flag if all ports up or timeout */
3173                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3174                         print_flag = 1;
3175                 }
3176
3177                 if (lsc_interrupt)
3178                         break;
3179         }
3180 }
3181
3182 static void
3183 rmv_port_callback(void *arg)
3184 {
3185         int need_to_start = 0;
3186         int org_no_link_check = no_link_check;
3187         portid_t port_id = (intptr_t)arg;
3188         struct rte_eth_dev_info dev_info;
3189         int ret;
3190
3191         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3192
3193         if (!test_done && port_is_forwarding(port_id)) {
3194                 need_to_start = 1;
3195                 stop_packet_forwarding();
3196         }
3197         no_link_check = 1;
3198         stop_port(port_id);
3199         no_link_check = org_no_link_check;
3200
3201         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3202         if (ret != 0)
3203                 TESTPMD_LOG(ERR,
3204                         "Failed to get device info for port %d, not detaching\n",
3205                         port_id);
3206         else {
3207                 struct rte_device *device = dev_info.device;
3208                 close_port(port_id);
3209                 detach_device(device); /* might be already removed or have more ports */
3210         }
3211         if (need_to_start)
3212                 start_packet_forwarding(0);
3213 }
3214
3215 /* This function is used by the interrupt thread */
3216 static int
3217 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3218                   void *ret_param)
3219 {
3220         RTE_SET_USED(param);
3221         RTE_SET_USED(ret_param);
3222
3223         if (type >= RTE_ETH_EVENT_MAX) {
3224                 fprintf(stderr,
3225                         "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3226                         port_id, __func__, type);
3227                 fflush(stderr);
3228         } else if (event_print_mask & (UINT32_C(1) << type)) {
3229                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3230                         eth_event_desc[type]);
3231                 fflush(stdout);
3232         }
3233
3234         switch (type) {
3235         case RTE_ETH_EVENT_NEW:
3236                 ports[port_id].need_setup = 1;
3237                 ports[port_id].port_status = RTE_PORT_HANDLING;
3238                 break;
3239         case RTE_ETH_EVENT_INTR_RMV:
3240                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3241                         break;
3242                 if (rte_eal_alarm_set(100000,
3243                                 rmv_port_callback, (void *)(intptr_t)port_id))
3244                         fprintf(stderr,
3245                                 "Could not set up deferred device removal\n");
3246                 break;
3247         case RTE_ETH_EVENT_DESTROY:
3248                 ports[port_id].port_status = RTE_PORT_CLOSED;
3249                 printf("Port %u is closed\n", port_id);
3250                 break;
3251         default:
3252                 break;
3253         }
3254         return 0;
3255 }
3256
3257 static int
3258 register_eth_event_callback(void)
3259 {
3260         int ret;
3261         enum rte_eth_event_type event;
3262
3263         for (event = RTE_ETH_EVENT_UNKNOWN;
3264                         event < RTE_ETH_EVENT_MAX; event++) {
3265                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3266                                 event,
3267                                 eth_event_callback,
3268                                 NULL);
3269                 if (ret != 0) {
3270                         TESTPMD_LOG(ERR, "Failed to register callback for "
3271                                         "%s event\n", eth_event_desc[event]);
3272                         return -1;
3273                 }
3274         }
3275
3276         return 0;
3277 }
3278
3279 /* This function is used by the interrupt thread */
3280 static void
3281 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3282                              __rte_unused void *arg)
3283 {
3284         uint16_t port_id;
3285         int ret;
3286
3287         if (type >= RTE_DEV_EVENT_MAX) {
3288                 fprintf(stderr, "%s called upon invalid event %d\n",
3289                         __func__, type);
3290                 fflush(stderr);
3291         }
3292
3293         switch (type) {
3294         case RTE_DEV_EVENT_REMOVE:
3295                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3296                         device_name);
3297                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3298                 if (ret) {
3299                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3300                                 device_name);
3301                         return;
3302                 }
3303                 /*
3304                  * Because the user's callback is invoked in eal interrupt
3305                  * callback, the interrupt callback need to be finished before
3306                  * it can be unregistered when detaching device. So finish
3307                  * callback soon and use a deferred removal to detach device
3308                  * is need. It is a workaround, once the device detaching be
3309                  * moved into the eal in the future, the deferred removal could
3310                  * be deleted.
3311                  */
3312                 if (rte_eal_alarm_set(100000,
3313                                 rmv_port_callback, (void *)(intptr_t)port_id))
3314                         RTE_LOG(ERR, EAL,
3315                                 "Could not set up deferred device removal\n");
3316                 break;
3317         case RTE_DEV_EVENT_ADD:
3318                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3319                         device_name);
3320                 /* TODO: After finish kernel driver binding,
3321                  * begin to attach port.
3322                  */
3323                 break;
3324         default:
3325                 break;
3326         }
3327 }
3328
3329 static void
3330 rxtx_port_config(struct rte_port *port)
3331 {
3332         uint16_t qid;
3333         uint64_t offloads;
3334
3335         for (qid = 0; qid < nb_rxq; qid++) {
3336                 offloads = port->rx_conf[qid].offloads;
3337                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3338                 if (offloads != 0)
3339                         port->rx_conf[qid].offloads = offloads;
3340
3341                 /* Check if any Rx parameters have been passed */
3342                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3343                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3344
3345                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3346                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3347
3348                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3349                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3350
3351                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3352                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3353
3354                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3355                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3356
3357                 port->nb_rx_desc[qid] = nb_rxd;
3358         }
3359
3360         for (qid = 0; qid < nb_txq; qid++) {
3361                 offloads = port->tx_conf[qid].offloads;
3362                 port->tx_conf[qid] = port->dev_info.default_txconf;
3363                 if (offloads != 0)
3364                         port->tx_conf[qid].offloads = offloads;
3365
3366                 /* Check if any Tx parameters have been passed */
3367                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3368                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3369
3370                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3371                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3372
3373                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3374                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3375
3376                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3377                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3378
3379                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3380                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3381
3382                 port->nb_tx_desc[qid] = nb_txd;
3383         }
3384 }
3385
3386 /*
3387  * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3388  * MTU is also aligned if JUMBO_FRAME offload is not set.
3389  *
3390  * port->dev_info should be set before calling this function.
3391  *
3392  * return 0 on success, negative on error
3393  */
3394 int
3395 update_jumbo_frame_offload(portid_t portid)
3396 {
3397         struct rte_port *port = &ports[portid];
3398         uint32_t eth_overhead;
3399         uint64_t rx_offloads;
3400         int ret;
3401         bool on;
3402
3403         /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3404         if (port->dev_info.max_mtu != UINT16_MAX &&
3405             port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3406                 eth_overhead = port->dev_info.max_rx_pktlen -
3407                                 port->dev_info.max_mtu;
3408         else
3409                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3410
3411         rx_offloads = port->dev_conf.rxmode.offloads;
3412
3413         /* Default config value is 0 to use PMD specific overhead */
3414         if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3415                 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3416
3417         if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3418                 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3419                 on = false;
3420         } else {
3421                 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3422                         fprintf(stderr,
3423                                 "Frame size (%u) is not supported by port %u\n",
3424                                 port->dev_conf.rxmode.max_rx_pkt_len,
3425                                 portid);
3426                         return -1;
3427                 }
3428                 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3429                 on = true;
3430         }
3431
3432         if (rx_offloads != port->dev_conf.rxmode.offloads) {
3433                 uint16_t qid;
3434
3435                 port->dev_conf.rxmode.offloads = rx_offloads;
3436
3437                 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3438                 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3439                         if (on)
3440                                 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3441                         else
3442                                 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3443                 }
3444         }
3445
3446         /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3447          * if unset do it here
3448          */
3449         if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3450                 ret = rte_eth_dev_set_mtu(portid,
3451                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3452                 if (ret)
3453                         fprintf(stderr,
3454                                 "Failed to set MTU to %u for port %u\n",
3455                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3456                                 portid);
3457         }
3458
3459         return 0;
3460 }
3461
3462 void
3463 init_port_config(void)
3464 {
3465         portid_t pid;
3466         struct rte_port *port;
3467         int ret;
3468
3469         RTE_ETH_FOREACH_DEV(pid) {
3470                 port = &ports[pid];
3471                 port->dev_conf.fdir_conf = fdir_conf;
3472
3473                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3474                 if (ret != 0)
3475                         return;
3476
3477                 if (nb_rxq > 1) {
3478                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3479                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3480                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3481                 } else {
3482                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3483                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3484                 }
3485
3486                 if (port->dcb_flag == 0) {
3487                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3488                                 port->dev_conf.rxmode.mq_mode =
3489                                         (enum rte_eth_rx_mq_mode)
3490                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3491                         else
3492                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3493                 }
3494
3495                 rxtx_port_config(port);
3496
3497                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3498                 if (ret != 0)
3499                         return;
3500
3501 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3502                 rte_pmd_ixgbe_bypass_init(pid);
3503 #endif
3504
3505                 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3506                         port->dev_conf.intr_conf.lsc = 1;
3507                 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3508                         port->dev_conf.intr_conf.rmv = 1;
3509         }
3510 }
3511
3512 void set_port_slave_flag(portid_t slave_pid)
3513 {
3514         struct rte_port *port;
3515
3516         port = &ports[slave_pid];
3517         port->slave_flag = 1;
3518 }
3519
3520 void clear_port_slave_flag(portid_t slave_pid)
3521 {
3522         struct rte_port *port;
3523
3524         port = &ports[slave_pid];
3525         port->slave_flag = 0;
3526 }
3527
3528 uint8_t port_is_bonding_slave(portid_t slave_pid)
3529 {
3530         struct rte_port *port;
3531         struct rte_eth_dev_info dev_info;
3532         int ret;
3533
3534         port = &ports[slave_pid];
3535         ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3536         if (ret != 0) {
3537                 TESTPMD_LOG(ERR,
3538                         "Failed to get device info for port id %d,"
3539                         "cannot determine if the port is a bonded slave",
3540                         slave_pid);
3541                 return 0;
3542         }
3543         if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3544                 return 1;
3545         return 0;
3546 }
3547
3548 const uint16_t vlan_tags[] = {
3549                 0,  1,  2,  3,  4,  5,  6,  7,
3550                 8,  9, 10, 11,  12, 13, 14, 15,
3551                 16, 17, 18, 19, 20, 21, 22, 23,
3552                 24, 25, 26, 27, 28, 29, 30, 31
3553 };
3554
3555 static  int
3556 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3557                  enum dcb_mode_enable dcb_mode,
3558                  enum rte_eth_nb_tcs num_tcs,
3559                  uint8_t pfc_en)
3560 {
3561         uint8_t i;
3562         int32_t rc;
3563         struct rte_eth_rss_conf rss_conf;
3564
3565         /*
3566          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3567          * given above, and the number of traffic classes available for use.
3568          */
3569         if (dcb_mode == DCB_VT_ENABLED) {
3570                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3571                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3572                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3573                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3574
3575                 /* VMDQ+DCB RX and TX configurations */
3576                 vmdq_rx_conf->enable_default_pool = 0;
3577                 vmdq_rx_conf->default_pool = 0;
3578                 vmdq_rx_conf->nb_queue_pools =
3579                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3580                 vmdq_tx_conf->nb_queue_pools =
3581                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3582
3583                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3584                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3585                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3586                         vmdq_rx_conf->pool_map[i].pools =
3587                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3588                 }
3589                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3590                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3591                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3592                 }
3593
3594                 /* set DCB mode of RX and TX of multiple queues */
3595                 eth_conf->rxmode.mq_mode =
3596                                 (enum rte_eth_rx_mq_mode)
3597                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3598                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3599         } else {
3600                 struct rte_eth_dcb_rx_conf *rx_conf =
3601                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3602                 struct rte_eth_dcb_tx_conf *tx_conf =
3603                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3604
3605                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3606
3607                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3608                 if (rc != 0)
3609                         return rc;
3610
3611                 rx_conf->nb_tcs = num_tcs;
3612                 tx_conf->nb_tcs = num_tcs;
3613
3614                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3615                         rx_conf->dcb_tc[i] = i % num_tcs;
3616                         tx_conf->dcb_tc[i] = i % num_tcs;
3617                 }
3618
3619                 eth_conf->rxmode.mq_mode =
3620                                 (enum rte_eth_rx_mq_mode)
3621                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3622                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3623                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3624         }
3625
3626         if (pfc_en)
3627                 eth_conf->dcb_capability_en =
3628                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3629         else
3630                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3631
3632         return 0;
3633 }
3634
3635 int
3636 init_port_dcb_config(portid_t pid,
3637                      enum dcb_mode_enable dcb_mode,
3638                      enum rte_eth_nb_tcs num_tcs,
3639                      uint8_t pfc_en)
3640 {
3641         struct rte_eth_conf port_conf;
3642         struct rte_port *rte_port;
3643         int retval;
3644         uint16_t i;
3645
3646         rte_port = &ports[pid];
3647
3648         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3649
3650         port_conf.rxmode = rte_port->dev_conf.rxmode;
3651         port_conf.txmode = rte_port->dev_conf.txmode;
3652
3653         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3654         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3655         if (retval < 0)
3656                 return retval;
3657         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3658
3659         /* re-configure the device . */
3660         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3661         if (retval < 0)
3662                 return retval;
3663
3664         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3665         if (retval != 0)
3666                 return retval;
3667
3668         /* If dev_info.vmdq_pool_base is greater than 0,
3669          * the queue id of vmdq pools is started after pf queues.
3670          */
3671         if (dcb_mode == DCB_VT_ENABLED &&
3672             rte_port->dev_info.vmdq_pool_base > 0) {
3673                 fprintf(stderr,
3674                         "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n",
3675                         pid);
3676                 return -1;
3677         }
3678
3679         /* Assume the ports in testpmd have the same dcb capability
3680          * and has the same number of rxq and txq in dcb mode
3681          */
3682         if (dcb_mode == DCB_VT_ENABLED) {
3683                 if (rte_port->dev_info.max_vfs > 0) {
3684                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3685                         nb_txq = rte_port->dev_info.nb_tx_queues;
3686                 } else {
3687                         nb_rxq = rte_port->dev_info.max_rx_queues;
3688                         nb_txq = rte_port->dev_info.max_tx_queues;
3689                 }
3690         } else {
3691                 /*if vt is disabled, use all pf queues */
3692                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3693                         nb_rxq = rte_port->dev_info.max_rx_queues;
3694                         nb_txq = rte_port->dev_info.max_tx_queues;
3695                 } else {
3696                         nb_rxq = (queueid_t)num_tcs;
3697                         nb_txq = (queueid_t)num_tcs;
3698
3699                 }
3700         }
3701         rx_free_thresh = 64;
3702
3703         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3704
3705         rxtx_port_config(rte_port);
3706         /* VLAN filter */
3707         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3708         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3709                 rx_vft_set(pid, vlan_tags[i], 1);
3710
3711         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3712         if (retval != 0)
3713                 return retval;
3714
3715         rte_port->dcb_flag = 1;
3716
3717         /* Enter DCB configuration status */
3718         dcb_config = 1;
3719
3720         return 0;
3721 }
3722
3723 static void
3724 init_port(void)
3725 {
3726         int i;
3727
3728         /* Configuration of Ethernet ports. */
3729         ports = rte_zmalloc("testpmd: ports",
3730                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3731                             RTE_CACHE_LINE_SIZE);
3732         if (ports == NULL) {
3733                 rte_exit(EXIT_FAILURE,
3734                                 "rte_zmalloc(%d struct rte_port) failed\n",
3735                                 RTE_MAX_ETHPORTS);
3736         }
3737         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3738                 LIST_INIT(&ports[i].flow_tunnel_list);
3739         /* Initialize ports NUMA structures */
3740         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3741         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3742         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3743 }
3744
3745 static void
3746 force_quit(void)
3747 {
3748         pmd_test_exit();
3749         prompt_exit();
3750 }
3751
3752 static void
3753 print_stats(void)
3754 {
3755         uint8_t i;
3756         const char clr[] = { 27, '[', '2', 'J', '\0' };
3757         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3758
3759         /* Clear screen and move to top left */
3760         printf("%s%s", clr, top_left);
3761
3762         printf("\nPort statistics ====================================");
3763         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3764                 nic_stats_display(fwd_ports_ids[i]);
3765
3766         fflush(stdout);
3767 }
3768
3769 static void
3770 signal_handler(int signum)
3771 {
3772         if (signum == SIGINT || signum == SIGTERM) {
3773                 fprintf(stderr, "\nSignal %d received, preparing to exit...\n",
3774                         signum);
3775 #ifdef RTE_LIB_PDUMP
3776                 /* uninitialize packet capture framework */
3777                 rte_pdump_uninit();
3778 #endif
3779 #ifdef RTE_LIB_LATENCYSTATS
3780                 if (latencystats_enabled != 0)
3781                         rte_latencystats_uninit();
3782 #endif
3783                 force_quit();
3784                 /* Set flag to indicate the force termination. */
3785                 f_quit = 1;
3786                 /* exit with the expected status */
3787 #ifndef RTE_EXEC_ENV_WINDOWS
3788                 signal(signum, SIG_DFL);
3789                 kill(getpid(), signum);
3790 #endif
3791         }
3792 }
3793
3794 int
3795 main(int argc, char** argv)
3796 {
3797         int diag;
3798         portid_t port_id;
3799         uint16_t count;
3800         int ret;
3801
3802         signal(SIGINT, signal_handler);
3803         signal(SIGTERM, signal_handler);
3804
3805         testpmd_logtype = rte_log_register("testpmd");
3806         if (testpmd_logtype < 0)
3807                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3808         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3809
3810         diag = rte_eal_init(argc, argv);
3811         if (diag < 0)
3812                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3813                          rte_strerror(rte_errno));
3814
3815         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3816                 rte_exit(EXIT_FAILURE,
3817                          "Secondary process type not supported.\n");
3818
3819         ret = register_eth_event_callback();
3820         if (ret != 0)
3821                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3822
3823 #ifdef RTE_LIB_PDUMP
3824         /* initialize packet capture framework */
3825         rte_pdump_init();
3826 #endif
3827
3828         count = 0;
3829         RTE_ETH_FOREACH_DEV(port_id) {
3830                 ports_ids[count] = port_id;
3831                 count++;
3832         }
3833         nb_ports = (portid_t) count;
3834         if (nb_ports == 0)
3835                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3836
3837         /* allocate port structures, and init them */
3838         init_port();
3839
3840         set_def_fwd_config();
3841         if (nb_lcores == 0)
3842                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3843                          "Check the core mask argument\n");
3844
3845         /* Bitrate/latency stats disabled by default */
3846 #ifdef RTE_LIB_BITRATESTATS
3847         bitrate_enabled = 0;
3848 #endif
3849 #ifdef RTE_LIB_LATENCYSTATS
3850         latencystats_enabled = 0;
3851 #endif
3852
3853         /* on FreeBSD, mlockall() is disabled by default */
3854 #ifdef RTE_EXEC_ENV_FREEBSD
3855         do_mlockall = 0;
3856 #else
3857         do_mlockall = 1;
3858 #endif
3859
3860         argc -= diag;
3861         argv += diag;
3862         if (argc > 1)
3863                 launch_args_parse(argc, argv);
3864
3865 #ifndef RTE_EXEC_ENV_WINDOWS
3866         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3867                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3868                         strerror(errno));
3869         }
3870 #endif
3871
3872         if (tx_first && interactive)
3873                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3874                                 "interactive mode.\n");
3875
3876         if (tx_first && lsc_interrupt) {
3877                 fprintf(stderr,
3878                         "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n");
3879                 lsc_interrupt = 0;
3880         }
3881
3882         if (!nb_rxq && !nb_txq)
3883                 fprintf(stderr,
3884                         "Warning: Either rx or tx queues should be non-zero\n");
3885
3886         if (nb_rxq > 1 && nb_rxq > nb_txq)
3887                 fprintf(stderr,
3888                         "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n",
3889                         nb_rxq, nb_txq);
3890
3891         init_config();
3892
3893         if (hot_plug) {
3894                 ret = rte_dev_hotplug_handle_enable();
3895                 if (ret) {
3896                         RTE_LOG(ERR, EAL,
3897                                 "fail to enable hotplug handling.");
3898                         return -1;
3899                 }
3900
3901                 ret = rte_dev_event_monitor_start();
3902                 if (ret) {
3903                         RTE_LOG(ERR, EAL,
3904                                 "fail to start device event monitoring.");
3905                         return -1;
3906                 }
3907
3908                 ret = rte_dev_event_callback_register(NULL,
3909                         dev_event_callback, NULL);
3910                 if (ret) {
3911                         RTE_LOG(ERR, EAL,
3912                                 "fail  to register device event callback\n");
3913                         return -1;
3914                 }
3915         }
3916
3917         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3918                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3919
3920         /* set all ports to promiscuous mode by default */
3921         RTE_ETH_FOREACH_DEV(port_id) {
3922                 ret = rte_eth_promiscuous_enable(port_id);
3923                 if (ret != 0)
3924                         fprintf(stderr,
3925                                 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
3926                                 port_id, rte_strerror(-ret));
3927         }
3928
3929         /* Init metrics library */
3930         rte_metrics_init(rte_socket_id());
3931
3932 #ifdef RTE_LIB_LATENCYSTATS
3933         if (latencystats_enabled != 0) {
3934                 int ret = rte_latencystats_init(1, NULL);
3935                 if (ret)
3936                         fprintf(stderr,
3937                                 "Warning: latencystats init() returned error %d\n",
3938                                 ret);
3939                 fprintf(stderr, "Latencystats running on lcore %d\n",
3940                         latencystats_lcore_id);
3941         }
3942 #endif
3943
3944         /* Setup bitrate stats */
3945 #ifdef RTE_LIB_BITRATESTATS
3946         if (bitrate_enabled != 0) {
3947                 bitrate_data = rte_stats_bitrate_create();
3948                 if (bitrate_data == NULL)
3949                         rte_exit(EXIT_FAILURE,
3950                                 "Could not allocate bitrate data.\n");
3951                 rte_stats_bitrate_reg(bitrate_data);
3952         }
3953 #endif
3954
3955 #ifdef RTE_LIB_CMDLINE
3956         if (strlen(cmdline_filename) != 0)
3957                 cmdline_read_from_file(cmdline_filename);
3958
3959         if (interactive == 1) {
3960                 if (auto_start) {
3961                         printf("Start automatic packet forwarding\n");
3962                         start_packet_forwarding(0);
3963                 }
3964                 prompt();
3965                 pmd_test_exit();
3966         } else
3967 #endif
3968         {
3969                 char c;
3970                 int rc;
3971
3972                 f_quit = 0;
3973
3974                 printf("No commandline core given, start packet forwarding\n");
3975                 start_packet_forwarding(tx_first);
3976                 if (stats_period != 0) {
3977                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3978                         uint64_t timer_period;
3979
3980                         /* Convert to number of cycles */
3981                         timer_period = stats_period * rte_get_timer_hz();
3982
3983                         while (f_quit == 0) {
3984                                 cur_time = rte_get_timer_cycles();
3985                                 diff_time += cur_time - prev_time;
3986
3987                                 if (diff_time >= timer_period) {
3988                                         print_stats();
3989                                         /* Reset the timer */
3990                                         diff_time = 0;
3991                                 }
3992                                 /* Sleep to avoid unnecessary checks */
3993                                 prev_time = cur_time;
3994                                 rte_delay_us_sleep(US_PER_S);
3995                         }
3996                 }
3997
3998                 printf("Press enter to exit\n");
3999                 rc = read(0, &c, 1);
4000                 pmd_test_exit();
4001                 if (rc < 0)
4002                         return 1;
4003         }
4004
4005         ret = rte_eal_cleanup();
4006         if (ret != 0)
4007                 rte_exit(EXIT_FAILURE,
4008                          "EAL cleanup failed: %s\n", strerror(-ret));
4009
4010         return EXIT_SUCCESS;
4011 }