app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_malloc_heap.h>
  31 #include <rte_memory.h>
  32 #include <rte_memcpy.h>
  33 #include <rte_launch.h>
  34 #include <rte_eal.h>
  35 #include <rte_alarm.h>
  36 #include <rte_per_lcore.h>
  37 #include <rte_lcore.h>
  38 #include <rte_atomic.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_LIBRTE_IXGBE_PMD
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIBRTE_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #include <rte_metrics.h>
  58 #ifdef RTE_LIBRTE_BITRATE
  59 #include <rte_bitrate.h>
  60 #endif
  61 #ifdef RTE_LIBRTE_LATENCY_STATS
  62 #include <rte_latencystats.h>
  63 #endif
  64
  65 #include "testpmd.h"
  66
  67 #ifndef MAP_HUGETLB
  68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  69 #define HUGE_FLAG (0x40000)
  70 #else
  71 #define HUGE_FLAG MAP_HUGETLB
  72 #endif
  73
  74 #ifndef MAP_HUGE_SHIFT
  75 /* older kernels (or FreeBSD) will not have this define */
  76 #define HUGE_SHIFT (26)
  77 #else
  78 #define HUGE_SHIFT MAP_HUGE_SHIFT
  79 #endif
  80
  81 #define EXTMEM_HEAP_NAME "extmem"
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192 uint16_t mempool_flags;
 193
 194 struct fwd_config cur_fwd_config;
 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 196 uint32_t retry_enabled;
 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 199
 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 201 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 202                                       * specified on command-line. */
 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 204
 205 /*
 206  * In container, it cannot terminate the process which running with 'stats-period'
 207  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 208  */
 209 uint8_t f_quit;
 210
 211 /*
 212  * Configuration of packet segments used by the "txonly" processing engine.
 213  */
 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 216         TXONLY_DEF_PACKET_LEN,
 217 };
 218 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 219
 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 221 /**< Split policy for packets to TX. */
 222
 223 uint8_t txonly_multi_flow;
 224 /**< Whether multiple flows are generated in TXONLY mode. */
 225
 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 228
 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 230 uint8_t dcb_config = 0;
 231
 232 /* Whether the dcb is in testing status */
 233 uint8_t dcb_test = 0;
 234
 235 /*
 236  * Configurable number of RX/TX queues.
 237  */
 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 240
 241 /*
 242  * Configurable number of RX/TX ring descriptors.
 243  * Defaults are supplied by drivers via ethdev.
 244  */
 245 #define RTE_TEST_RX_DESC_DEFAULT 0
 246 #define RTE_TEST_TX_DESC_DEFAULT 0
 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 249
 250 #define RTE_PMD_PARAM_UNSET -1
 251 /*
 252  * Configurable values of RX and TX ring threshold registers.
 253  */
 254
 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 258
 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 262
 263 /*
 264  * Configurable value of RX free threshold.
 265  */
 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 267
 268 /*
 269  * Configurable value of RX drop enable.
 270  */
 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 272
 273 /*
 274  * Configurable value of TX free threshold.
 275  */
 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of TX RS bit threshold.
 280  */
 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of buffered packets before sending.
 285  */
 286 uint16_t noisy_tx_sw_bufsz;
 287
 288 /*
 289  * Configurable value of packet buffer timeout.
 290  */
 291 uint16_t noisy_tx_sw_buf_flush_time;
 292
 293 /*
 294  * Configurable value for size of VNF internal memory area
 295  * used for simulating noisy neighbour behaviour
 296  */
 297 uint64_t noisy_lkup_mem_sz;
 298
 299 /*
 300  * Configurable value of number of random writes done in
 301  * VNF simulation memory area.
 302  */
 303 uint64_t noisy_lkup_num_writes;
 304
 305 /*
 306  * Configurable value of number of random reads done in
 307  * VNF simulation memory area.
 308  */
 309 uint64_t noisy_lkup_num_reads;
 310
 311 /*
 312  * Configurable value of number of random reads/writes done in
 313  * VNF simulation memory area.
 314  */
 315 uint64_t noisy_lkup_num_reads_writes;
 316
 317 /*
 318  * Receive Side Scaling (RSS) configuration.
 319  */
 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 321
 322 /*
 323  * Port topology configuration
 324  */
 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 326
 327 /*
 328  * Avoids to flush all the RX streams before starts forwarding.
 329  */
 330 uint8_t no_flush_rx = 0; /* flush by default */
 331
 332 /*
 333  * Flow API isolated mode.
 334  */
 335 uint8_t flow_isolate_all;
 336
 337 /*
 338  * Avoids to check link status when starting/stopping a port.
 339  */
 340 uint8_t no_link_check = 0; /* check by default */
 341
 342 /*
 343  * Don't automatically start all ports in interactive mode.
 344  */
 345 uint8_t no_device_start = 0;
 346
 347 /*
 348  * Enable link status change notification
 349  */
 350 uint8_t lsc_interrupt = 1; /* enabled by default */
 351
 352 /*
 353  * Enable device removal notification.
 354  */
 355 uint8_t rmv_interrupt = 1; /* enabled by default */
 356
 357 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 358
 359 /* After attach, port setup is called on event or by iterator */
 360 bool setup_on_probe_event = true;
 361
 362 /* Pretty printing of ethdev events */
 363 static const char * const eth_event_desc[] = {
 364         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 365         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 366         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 367         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 368         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 369         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 370         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 371         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 372         [RTE_ETH_EVENT_NEW] = "device probed",
 373         [RTE_ETH_EVENT_DESTROY] = "device released",
 374         [RTE_ETH_EVENT_MAX] = NULL,
 375 };
 376
 377 /*
 378  * Display or mask ether events
 379  * Default to all events except VF_MBOX
 380  */
 381 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 382                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 383                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 384                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 385                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 386                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 387                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 388 /*
 389  * Decide if all memory are locked for performance.
 390  */
 391 int do_mlockall = 0;
 392
 393 /*
 394  * NIC bypass mode configuration options.
 395  */
 396
 397 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 398 /* The NIC bypass watchdog timeout. */
 399 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 400 #endif
 401
 402
 403 #ifdef RTE_LIBRTE_LATENCY_STATS
 404
 405 /*
 406  * Set when latency stats is enabled in the commandline
 407  */
 408 uint8_t latencystats_enabled;
 409
 410 /*
 411  * Lcore ID to serive latency statistics.
 412  */
 413 lcoreid_t latencystats_lcore_id = -1;
 414
 415 #endif
 416
 417 /*
 418  * Ethernet device configuration.
 419  */
 420 struct rte_eth_rxmode rx_mode = {
 421         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 422                 /**< Default maximum frame length. */
 423 };
 424
 425 struct rte_eth_txmode tx_mode = {
 426         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 427 };
 428
 429 struct rte_fdir_conf fdir_conf = {
 430         .mode = RTE_FDIR_MODE_NONE,
 431         .pballoc = RTE_FDIR_PBALLOC_64K,
 432         .status = RTE_FDIR_REPORT_STATUS,
 433         .mask = {
 434                 .vlan_tci_mask = 0xFFEF,
 435                 .ipv4_mask     = {
 436                         .src_ip = 0xFFFFFFFF,
 437                         .dst_ip = 0xFFFFFFFF,
 438                 },
 439                 .ipv6_mask     = {
 440                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 441                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 442                 },
 443                 .src_port_mask = 0xFFFF,
 444                 .dst_port_mask = 0xFFFF,
 445                 .mac_addr_byte_mask = 0xFF,
 446                 .tunnel_type_mask = 1,
 447                 .tunnel_id_mask = 0xFFFFFFFF,
 448         },
 449         .drop_queue = 127,
 450 };
 451
 452 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 453
 454 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 455 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 456
 457 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 458 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 459
 460 uint16_t nb_tx_queue_stats_mappings = 0;
 461 uint16_t nb_rx_queue_stats_mappings = 0;
 462
 463 /*
 464  * Display zero values by default for xstats
 465  */
 466 uint8_t xstats_hide_zero;
 467
 468 unsigned int num_sockets = 0;
 469 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 470
 471 #ifdef RTE_LIBRTE_BITRATE
 472 /* Bitrate statistics */
 473 struct rte_stats_bitrates *bitrate_data;
 474 lcoreid_t bitrate_lcore_id;
 475 uint8_t bitrate_enabled;
 476 #endif
 477
 478 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 479 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 480
 481 /* Forward function declarations */
 482 static void setup_attached_port(portid_t pi);
 483 static void map_port_queue_stats_mapping_registers(portid_t pi,
 484                                                    struct rte_port *port);
 485 static void check_all_ports_link_status(uint32_t port_mask);
 486 static int eth_event_callback(portid_t port_id,
 487                               enum rte_eth_event_type type,
 488                               void *param, void *ret_param);
 489 static void dev_event_callback(const char *device_name,
 490                                 enum rte_dev_event_type type,
 491                                 void *param);
 492
 493 /*
 494  * Check if all the ports are started.
 495  * If yes, return positive value. If not, return zero.
 496  */
 497 static int all_ports_started(void);
 498
 499 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 500 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 501
 502 /*
 503  * Helper function to check if socket is already discovered.
 504  * If yes, return positive value. If not, return zero.
 505  */
 506 int
 507 new_socket_id(unsigned int socket_id)
 508 {
 509         unsigned int i;
 510
 511         for (i = 0; i < num_sockets; i++) {
 512                 if (socket_ids[i] == socket_id)
 513                         return 0;
 514         }
 515         return 1;
 516 }
 517
 518 /*
 519  * Setup default configuration.
 520  */
 521 static void
 522 set_default_fwd_lcores_config(void)
 523 {
 524         unsigned int i;
 525         unsigned int nb_lc;
 526         unsigned int sock_num;
 527
 528         nb_lc = 0;
 529         for (i = 0; i < RTE_MAX_LCORE; i++) {
 530                 if (!rte_lcore_is_enabled(i))
 531                         continue;
 532                 sock_num = rte_lcore_to_socket_id(i);
 533                 if (new_socket_id(sock_num)) {
 534                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 535                                 rte_exit(EXIT_FAILURE,
 536                                          "Total sockets greater than %u\n",
 537                                          RTE_MAX_NUMA_NODES);
 538                         }
 539                         socket_ids[num_sockets++] = sock_num;
 540                 }
 541                 if (i == rte_get_master_lcore())
 542                         continue;
 543                 fwd_lcores_cpuids[nb_lc++] = i;
 544         }
 545         nb_lcores = (lcoreid_t) nb_lc;
 546         nb_cfg_lcores = nb_lcores;
 547         nb_fwd_lcores = 1;
 548 }
 549
 550 static void
 551 set_def_peer_eth_addrs(void)
 552 {
 553         portid_t i;
 554
 555         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 556                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 557                 peer_eth_addrs[i].addr_bytes[5] = i;
 558         }
 559 }
 560
 561 static void
 562 set_default_fwd_ports_config(void)
 563 {
 564         portid_t pt_id;
 565         int i = 0;
 566
 567         RTE_ETH_FOREACH_DEV(pt_id) {
 568                 fwd_ports_ids[i++] = pt_id;
 569
 570                 /* Update sockets info according to the attached device */
 571                 int socket_id = rte_eth_dev_socket_id(pt_id);
 572                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 573                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 574                                 rte_exit(EXIT_FAILURE,
 575                                          "Total sockets greater than %u\n",
 576                                          RTE_MAX_NUMA_NODES);
 577                         }
 578                         socket_ids[num_sockets++] = socket_id;
 579                 }
 580         }
 581
 582         nb_cfg_ports = nb_ports;
 583         nb_fwd_ports = nb_ports;
 584 }
 585
 586 void
 587 set_def_fwd_config(void)
 588 {
 589         set_default_fwd_lcores_config();
 590         set_def_peer_eth_addrs();
 591         set_default_fwd_ports_config();
 592 }
 593
 594 /* extremely pessimistic estimation of memory required to create a mempool */
 595 static int
 596 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 597 {
 598         unsigned int n_pages, mbuf_per_pg, leftover;
 599         uint64_t total_mem, mbuf_mem, obj_sz;
 600
 601         /* there is no good way to predict how much space the mempool will
 602          * occupy because it will allocate chunks on the fly, and some of those
 603          * will come from default DPDK memory while some will come from our
 604          * external memory, so just assume 128MB will be enough for everyone.
 605          */
 606         uint64_t hdr_mem = 128 << 20;
 607
 608         /* account for possible non-contiguousness */
 609         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 610         if (obj_sz > pgsz) {
 611                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 612                 return -1;
 613         }
 614
 615         mbuf_per_pg = pgsz / obj_sz;
 616         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 617         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 618
 619         mbuf_mem = n_pages * pgsz;
 620
 621         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 622
 623         if (total_mem > SIZE_MAX) {
 624                 TESTPMD_LOG(ERR, "Memory size too big\n");
 625                 return -1;
 626         }
 627         *out = (size_t)total_mem;
 628
 629         return 0;
 630 }
 631
 632 static int
 633 pagesz_flags(uint64_t page_sz)
 634 {
 635         /* as per mmap() manpage, all page sizes are log2 of page size
 636          * shifted by MAP_HUGE_SHIFT
 637          */
 638         int log2 = rte_log2_u64(page_sz);
 639
 640         return (log2 << HUGE_SHIFT);
 641 }
 642
 643 static void *
 644 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 645 {
 646         void *addr;
 647         int flags;
 648
 649         /* allocate anonymous hugepages */
 650         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 651         if (huge)
 652                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 653
 654         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 655         if (addr == MAP_FAILED)
 656                 return NULL;
 657
 658         return addr;
 659 }
 660
 661 struct extmem_param {
 662         void *addr;
 663         size_t len;
 664         size_t pgsz;
 665         rte_iova_t *iova_table;
 666         unsigned int iova_table_len;
 667 };
 668
 669 static int
 670 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 671                 bool huge)
 672 {
 673         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 674                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 675         unsigned int cur_page, n_pages, pgsz_idx;
 676         size_t mem_sz, cur_pgsz;
 677         rte_iova_t *iovas = NULL;
 678         void *addr;
 679         int ret;
 680
 681         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 682                 /* skip anything that is too big */
 683                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 684                         continue;
 685
 686                 cur_pgsz = pgsizes[pgsz_idx];
 687
 688                 /* if we were told not to allocate hugepages, override */
 689                 if (!huge)
 690                         cur_pgsz = sysconf(_SC_PAGESIZE);
 691
 692                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 693                 if (ret < 0) {
 694                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 695                         return -1;
 696                 }
 697
 698                 /* allocate our memory */
 699                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 700
 701                 /* if we couldn't allocate memory with a specified page size,
 702                  * that doesn't mean we can't do it with other page sizes, so
 703                  * try another one.
 704                  */
 705                 if (addr == NULL)
 706                         continue;
 707
 708                 /* store IOVA addresses for every page in this memory area */
 709                 n_pages = mem_sz / cur_pgsz;
 710
 711                 iovas = malloc(sizeof(*iovas) * n_pages);
 712
 713                 if (iovas == NULL) {
 714                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 715                         goto fail;
 716                 }
 717                 /* lock memory if it's not huge pages */
 718                 if (!huge)
 719                         mlock(addr, mem_sz);
 720
 721                 /* populate IOVA addresses */
 722                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 723                         rte_iova_t iova;
 724                         size_t offset;
 725                         void *cur;
 726
 727                         offset = cur_pgsz * cur_page;
 728                         cur = RTE_PTR_ADD(addr, offset);
 729
 730                         /* touch the page before getting its IOVA */
 731                         *(volatile char *)cur = 0;
 732
 733                         iova = rte_mem_virt2iova(cur);
 734
 735                         iovas[cur_page] = iova;
 736                 }
 737
 738                 break;
 739         }
 740         /* if we couldn't allocate anything */
 741         if (iovas == NULL)
 742                 return -1;
 743
 744         param->addr = addr;
 745         param->len = mem_sz;
 746         param->pgsz = cur_pgsz;
 747         param->iova_table = iovas;
 748         param->iova_table_len = n_pages;
 749
 750         return 0;
 751 fail:
 752         if (iovas)
 753                 free(iovas);
 754         if (addr)
 755                 munmap(addr, mem_sz);
 756
 757         return -1;
 758 }
 759
 760 static int
 761 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 762 {
 763         struct extmem_param param;
 764         int socket_id, ret;
 765
 766         memset(&param, 0, sizeof(param));
 767
 768         /* check if our heap exists */
 769         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 770         if (socket_id < 0) {
 771                 /* create our heap */
 772                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 773                 if (ret < 0) {
 774                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 775                         return -1;
 776                 }
 777         }
 778
 779         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 780         if (ret < 0) {
 781                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 782                 return -1;
 783         }
 784
 785         /* we now have a valid memory area, so add it to heap */
 786         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 787                         param.addr, param.len, param.iova_table,
 788                         param.iova_table_len, param.pgsz);
 789
 790         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 791
 792         /* not needed any more */
 793         free(param.iova_table);
 794
 795         if (ret < 0) {
 796                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 797                 munmap(param.addr, param.len);
 798                 return -1;
 799         }
 800
 801         /* success */
 802
 803         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 804                         param.len >> 20);
 805
 806         return 0;
 807 }
 808 static void
 809 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 810              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 811 {
 812         uint16_t pid = 0;
 813         int ret;
 814
 815         RTE_ETH_FOREACH_DEV(pid) {
 816                 struct rte_eth_dev *dev =
 817                         &rte_eth_devices[pid];
 818
 819                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 820                                         memhdr->len);
 821                 if (ret) {
 822                         TESTPMD_LOG(DEBUG,
 823                                     "unable to DMA unmap addr 0x%p "
 824                                     "for device %s\n",
 825                                     memhdr->addr, dev->data->name);
 826                 }
 827         }
 828         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 829         if (ret) {
 830                 TESTPMD_LOG(DEBUG,
 831                             "unable to un-register addr 0x%p\n", memhdr->addr);
 832         }
 833 }
 834
 835 static void
 836 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 837            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 838 {
 839         uint16_t pid = 0;
 840         size_t page_size = sysconf(_SC_PAGESIZE);
 841         int ret;
 842
 843         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 844                                   page_size);
 845         if (ret) {
 846                 TESTPMD_LOG(DEBUG,
 847                             "unable to register addr 0x%p\n", memhdr->addr);
 848                 return;
 849         }
 850         RTE_ETH_FOREACH_DEV(pid) {
 851                 struct rte_eth_dev *dev =
 852                         &rte_eth_devices[pid];
 853
 854                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 855                                       memhdr->len);
 856                 if (ret) {
 857                         TESTPMD_LOG(DEBUG,
 858                                     "unable to DMA map addr 0x%p "
 859                                     "for device %s\n",
 860                                     memhdr->addr, dev->data->name);
 861                 }
 862         }
 863 }
 864
 865 /*
 866  * Configuration initialisation done once at init time.
 867  */
 868 static struct rte_mempool *
 869 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 870                  unsigned int socket_id)
 871 {
 872         char pool_name[RTE_MEMPOOL_NAMESIZE];
 873         struct rte_mempool *rte_mp = NULL;
 874         uint32_t mb_size;
 875
 876         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 877         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 878
 879         TESTPMD_LOG(INFO,
 880                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 881                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 882
 883         switch (mp_alloc_type) {
 884         case MP_ALLOC_NATIVE:
 885                 {
 886                         /* wrapper to rte_mempool_create() */
 887                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 888                                         rte_mbuf_best_mempool_ops());
 889                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 890                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 891                         break;
 892                 }
 893         case MP_ALLOC_ANON:
 894                 {
 895                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 896                                 mb_size, (unsigned int) mb_mempool_cache,
 897                                 sizeof(struct rte_pktmbuf_pool_private),
 898                                 socket_id, mempool_flags);
 899                         if (rte_mp == NULL)
 900                                 goto err;
 901
 902                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 903                                 rte_mempool_free(rte_mp);
 904                                 rte_mp = NULL;
 905                                 goto err;
 906                         }
 907                         rte_pktmbuf_pool_init(rte_mp, NULL);
 908                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 909                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 910                         break;
 911                 }
 912         case MP_ALLOC_XMEM:
 913         case MP_ALLOC_XMEM_HUGE:
 914                 {
 915                         int heap_socket;
 916                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 917
 918                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 919                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 920
 921                         heap_socket =
 922                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 923                         if (heap_socket < 0)
 924                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 925
 926                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 927                                         rte_mbuf_best_mempool_ops());
 928                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 929                                         mb_mempool_cache, 0, mbuf_seg_size,
 930                                         heap_socket);
 931                         break;
 932                 }
 933         default:
 934                 {
 935                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 936                 }
 937         }
 938
 939 err:
 940         if (rte_mp == NULL) {
 941                 rte_exit(EXIT_FAILURE,
 942                         "Creation of mbuf pool for socket %u failed: %s\n",
 943                         socket_id, rte_strerror(rte_errno));
 944         } else if (verbose_level > 0) {
 945                 rte_mempool_dump(stdout, rte_mp);
 946         }
 947         return rte_mp;
 948 }
 949
 950 /*
 951  * Check given socket id is valid or not with NUMA mode,
 952  * if valid, return 0, else return -1
 953  */
 954 static int
 955 check_socket_id(const unsigned int socket_id)
 956 {
 957         static int warning_once = 0;
 958
 959         if (new_socket_id(socket_id)) {
 960                 if (!warning_once && numa_support)
 961                         printf("Warning: NUMA should be configured manually by"
 962                                " using --port-numa-config and"
 963                                " --ring-numa-config parameters along with"
 964                                " --numa.\n");
 965                 warning_once = 1;
 966                 return -1;
 967         }
 968         return 0;
 969 }
 970
 971 /*
 972  * Get the allowed maximum number of RX queues.
 973  * *pid return the port id which has minimal value of
 974  * max_rx_queues in all ports.
 975  */
 976 queueid_t
 977 get_allowed_max_nb_rxq(portid_t *pid)
 978 {
 979         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
 980         bool max_rxq_valid = false;
 981         portid_t pi;
 982         struct rte_eth_dev_info dev_info;
 983
 984         RTE_ETH_FOREACH_DEV(pi) {
 985                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
 986                         continue;
 987
 988                 max_rxq_valid = true;
 989                 if (dev_info.max_rx_queues < allowed_max_rxq) {
 990                         allowed_max_rxq = dev_info.max_rx_queues;
 991                         *pid = pi;
 992                 }
 993         }
 994         return max_rxq_valid ? allowed_max_rxq : 0;
 995 }
 996
 997 /*
 998  * Check input rxq is valid or not.
 999  * If input rxq is not greater than any of maximum number
1000  * of RX queues of all ports, it is valid.
1001  * if valid, return 0, else return -1
1002  */
1003 int
1004 check_nb_rxq(queueid_t rxq)
1005 {
1006         queueid_t allowed_max_rxq;
1007         portid_t pid = 0;
1008
1009         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1010         if (rxq > allowed_max_rxq) {
1011                 printf("Fail: input rxq (%u) can't be greater "
1012                        "than max_rx_queues (%u) of port %u\n",
1013                        rxq,
1014                        allowed_max_rxq,
1015                        pid);
1016                 return -1;
1017         }
1018         return 0;
1019 }
1020
1021 /*
1022  * Get the allowed maximum number of TX queues.
1023  * *pid return the port id which has minimal value of
1024  * max_tx_queues in all ports.
1025  */
1026 queueid_t
1027 get_allowed_max_nb_txq(portid_t *pid)
1028 {
1029         queueid_t allowed_max_txq = MAX_QUEUE_ID;
1030         bool max_txq_valid = false;
1031         portid_t pi;
1032         struct rte_eth_dev_info dev_info;
1033
1034         RTE_ETH_FOREACH_DEV(pi) {
1035                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1036                         continue;
1037
1038                 max_txq_valid = true;
1039                 if (dev_info.max_tx_queues < allowed_max_txq) {
1040                         allowed_max_txq = dev_info.max_tx_queues;
1041                         *pid = pi;
1042                 }
1043         }
1044         return max_txq_valid ? allowed_max_txq : 0;
1045 }
1046
1047 /*
1048  * Check input txq is valid or not.
1049  * If input txq is not greater than any of maximum number
1050  * of TX queues of all ports, it is valid.
1051  * if valid, return 0, else return -1
1052  */
1053 int
1054 check_nb_txq(queueid_t txq)
1055 {
1056         queueid_t allowed_max_txq;
1057         portid_t pid = 0;
1058
1059         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1060         if (txq > allowed_max_txq) {
1061                 printf("Fail: input txq (%u) can't be greater "
1062                        "than max_tx_queues (%u) of port %u\n",
1063                        txq,
1064                        allowed_max_txq,
1065                        pid);
1066                 return -1;
1067         }
1068         return 0;
1069 }
1070
1071 static void
1072 init_config(void)
1073 {
1074         portid_t pid;
1075         struct rte_port *port;
1076         struct rte_mempool *mbp;
1077         unsigned int nb_mbuf_per_pool;
1078         lcoreid_t  lc_id;
1079         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1080         struct rte_gro_param gro_param;
1081         uint32_t gso_types;
1082         uint16_t data_size;
1083         bool warning = 0;
1084         int k;
1085         int ret;
1086
1087         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1088
1089         /* Configuration of logical cores. */
1090         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1091                                 sizeof(struct fwd_lcore *) * nb_lcores,
1092                                 RTE_CACHE_LINE_SIZE);
1093         if (fwd_lcores == NULL) {
1094                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1095                                                         "failed\n", nb_lcores);
1096         }
1097         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1098                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1099                                                sizeof(struct fwd_lcore),
1100                                                RTE_CACHE_LINE_SIZE);
1101                 if (fwd_lcores[lc_id] == NULL) {
1102                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1103                                                                 "failed\n");
1104                 }
1105                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1106         }
1107
1108         RTE_ETH_FOREACH_DEV(pid) {
1109                 port = &ports[pid];
1110                 /* Apply default TxRx configuration for all ports */
1111                 port->dev_conf.txmode = tx_mode;
1112                 port->dev_conf.rxmode = rx_mode;
1113
1114                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1115                 if (ret != 0)
1116                         rte_exit(EXIT_FAILURE,
1117                                  "rte_eth_dev_info_get() failed\n");
1118
1119                 if (!(port->dev_info.tx_offload_capa &
1120                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1121                         port->dev_conf.txmode.offloads &=
1122                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1123                 if (!(port->dev_info.tx_offload_capa &
1124                         DEV_TX_OFFLOAD_MATCH_METADATA))
1125                         port->dev_conf.txmode.offloads &=
1126                                 ~DEV_TX_OFFLOAD_MATCH_METADATA;
1127                 if (numa_support) {
1128                         if (port_numa[pid] != NUMA_NO_CONFIG)
1129                                 port_per_socket[port_numa[pid]]++;
1130                         else {
1131                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1132
1133                                 /*
1134                                  * if socket_id is invalid,
1135                                  * set to the first available socket.
1136                                  */
1137                                 if (check_socket_id(socket_id) < 0)
1138                                         socket_id = socket_ids[0];
1139                                 port_per_socket[socket_id]++;
1140                         }
1141                 }
1142
1143                 /* Apply Rx offloads configuration */
1144                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1145                         port->rx_conf[k].offloads =
1146                                 port->dev_conf.rxmode.offloads;
1147                 /* Apply Tx offloads configuration */
1148                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1149                         port->tx_conf[k].offloads =
1150                                 port->dev_conf.txmode.offloads;
1151
1152                 /* set flag to initialize port/queue */
1153                 port->need_reconfig = 1;
1154                 port->need_reconfig_queues = 1;
1155                 port->tx_metadata = 0;
1156
1157                 /* Check for maximum number of segments per MTU. Accordingly
1158                  * update the mbuf data size.
1159                  */
1160                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1161                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1162                         data_size = rx_mode.max_rx_pkt_len /
1163                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1164
1165                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1166                                                         mbuf_data_size) {
1167                                 mbuf_data_size = data_size +
1168                                                  RTE_PKTMBUF_HEADROOM;
1169                                 warning = 1;
1170                         }
1171                 }
1172         }
1173
1174         if (warning)
1175                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1176                             mbuf_data_size);
1177
1178         /*
1179          * Create pools of mbuf.
1180          * If NUMA support is disabled, create a single pool of mbuf in
1181          * socket 0 memory by default.
1182          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1183          *
1184          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1185          * nb_txd can be configured at run time.
1186          */
1187         if (param_total_num_mbufs)
1188                 nb_mbuf_per_pool = param_total_num_mbufs;
1189         else {
1190                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1191                         (nb_lcores * mb_mempool_cache) +
1192                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1193                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1194         }
1195
1196         if (numa_support) {
1197                 uint8_t i;
1198
1199                 for (i = 0; i < num_sockets; i++)
1200                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1201                                                        nb_mbuf_per_pool,
1202                                                        socket_ids[i]);
1203         } else {
1204                 if (socket_num == UMA_NO_CONFIG)
1205                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1206                                                        nb_mbuf_per_pool, 0);
1207                 else
1208                         mempools[socket_num] = mbuf_pool_create
1209                                                         (mbuf_data_size,
1210                                                          nb_mbuf_per_pool,
1211                                                          socket_num);
1212         }
1213
1214         init_port_config();
1215
1216         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1217                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1218         /*
1219          * Records which Mbuf pool to use by each logical core, if needed.
1220          */
1221         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1222                 mbp = mbuf_pool_find(
1223                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1224
1225                 if (mbp == NULL)
1226                         mbp = mbuf_pool_find(0);
1227                 fwd_lcores[lc_id]->mbp = mbp;
1228                 /* initialize GSO context */
1229                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1230                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1231                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1232                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1233                         RTE_ETHER_CRC_LEN;
1234                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1235         }
1236
1237         /* Configuration of packet forwarding streams. */
1238         if (init_fwd_streams() < 0)
1239                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1240
1241         fwd_config_setup();
1242
1243         /* create a gro context for each lcore */
1244         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1245         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1246         gro_param.max_item_per_flow = MAX_PKT_BURST;
1247         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1248                 gro_param.socket_id = rte_lcore_to_socket_id(
1249                                 fwd_lcores_cpuids[lc_id]);
1250                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1251                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1252                         rte_exit(EXIT_FAILURE,
1253                                         "rte_gro_ctx_create() failed\n");
1254                 }
1255         }
1256
1257 #if defined RTE_LIBRTE_PMD_SOFTNIC
1258         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1259                 RTE_ETH_FOREACH_DEV(pid) {
1260                         port = &ports[pid];
1261                         const char *driver = port->dev_info.driver_name;
1262
1263                         if (strcmp(driver, "net_softnic") == 0)
1264                                 port->softport.fwd_lcore_arg = fwd_lcores;
1265                 }
1266         }
1267 #endif
1268
1269 }
1270
1271
1272 void
1273 reconfig(portid_t new_port_id, unsigned socket_id)
1274 {
1275         struct rte_port *port;
1276         int ret;
1277
1278         /* Reconfiguration of Ethernet ports. */
1279         port = &ports[new_port_id];
1280
1281         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1282         if (ret != 0)
1283                 return;
1284
1285         /* set flag to initialize port/queue */
1286         port->need_reconfig = 1;
1287         port->need_reconfig_queues = 1;
1288         port->socket_id = socket_id;
1289
1290         init_port_config();
1291 }
1292
1293
1294 int
1295 init_fwd_streams(void)
1296 {
1297         portid_t pid;
1298         struct rte_port *port;
1299         streamid_t sm_id, nb_fwd_streams_new;
1300         queueid_t q;
1301
1302         /* set socket id according to numa or not */
1303         RTE_ETH_FOREACH_DEV(pid) {
1304                 port = &ports[pid];
1305                 if (nb_rxq > port->dev_info.max_rx_queues) {
1306                         printf("Fail: nb_rxq(%d) is greater than "
1307                                 "max_rx_queues(%d)\n", nb_rxq,
1308                                 port->dev_info.max_rx_queues);
1309                         return -1;
1310                 }
1311                 if (nb_txq > port->dev_info.max_tx_queues) {
1312                         printf("Fail: nb_txq(%d) is greater than "
1313                                 "max_tx_queues(%d)\n", nb_txq,
1314                                 port->dev_info.max_tx_queues);
1315                         return -1;
1316                 }
1317                 if (numa_support) {
1318                         if (port_numa[pid] != NUMA_NO_CONFIG)
1319                                 port->socket_id = port_numa[pid];
1320                         else {
1321                                 port->socket_id = rte_eth_dev_socket_id(pid);
1322
1323                                 /*
1324                                  * if socket_id is invalid,
1325                                  * set to the first available socket.
1326                                  */
1327                                 if (check_socket_id(port->socket_id) < 0)
1328                                         port->socket_id = socket_ids[0];
1329                         }
1330                 }
1331                 else {
1332                         if (socket_num == UMA_NO_CONFIG)
1333                                 port->socket_id = 0;
1334                         else
1335                                 port->socket_id = socket_num;
1336                 }
1337         }
1338
1339         q = RTE_MAX(nb_rxq, nb_txq);
1340         if (q == 0) {
1341                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1342                 return -1;
1343         }
1344         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1345         if (nb_fwd_streams_new == nb_fwd_streams)
1346                 return 0;
1347         /* clear the old */
1348         if (fwd_streams != NULL) {
1349                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1350                         if (fwd_streams[sm_id] == NULL)
1351                                 continue;
1352                         rte_free(fwd_streams[sm_id]);
1353                         fwd_streams[sm_id] = NULL;
1354                 }
1355                 rte_free(fwd_streams);
1356                 fwd_streams = NULL;
1357         }
1358
1359         /* init new */
1360         nb_fwd_streams = nb_fwd_streams_new;
1361         if (nb_fwd_streams) {
1362                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1363                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1364                         RTE_CACHE_LINE_SIZE);
1365                 if (fwd_streams == NULL)
1366                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1367                                  " (struct fwd_stream *)) failed\n",
1368                                  nb_fwd_streams);
1369
1370                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1371                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1372                                 " struct fwd_stream", sizeof(struct fwd_stream),
1373                                 RTE_CACHE_LINE_SIZE);
1374                         if (fwd_streams[sm_id] == NULL)
1375                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1376                                          "(struct fwd_stream) failed\n");
1377                 }
1378         }
1379
1380         return 0;
1381 }
1382
1383 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1384 static void
1385 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1386 {
1387         unsigned int total_burst;
1388         unsigned int nb_burst;
1389         unsigned int burst_stats[3];
1390         uint16_t pktnb_stats[3];
1391         uint16_t nb_pkt;
1392         int burst_percent[3];
1393
1394         /*
1395          * First compute the total number of packet bursts and the
1396          * two highest numbers of bursts of the same number of packets.
1397          */
1398         total_burst = 0;
1399         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1400         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1401         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1402                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1403                 if (nb_burst == 0)
1404                         continue;
1405                 total_burst += nb_burst;
1406                 if (nb_burst > burst_stats[0]) {
1407                         burst_stats[1] = burst_stats[0];
1408                         pktnb_stats[1] = pktnb_stats[0];
1409                         burst_stats[0] = nb_burst;
1410                         pktnb_stats[0] = nb_pkt;
1411                 } else if (nb_burst > burst_stats[1]) {
1412                         burst_stats[1] = nb_burst;
1413                         pktnb_stats[1] = nb_pkt;
1414                 }
1415         }
1416         if (total_burst == 0)
1417                 return;
1418         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1419         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1420                burst_percent[0], (int) pktnb_stats[0]);
1421         if (burst_stats[0] == total_burst) {
1422                 printf("]\n");
1423                 return;
1424         }
1425         if (burst_stats[0] + burst_stats[1] == total_burst) {
1426                 printf(" + %d%% of %d pkts]\n",
1427                        100 - burst_percent[0], pktnb_stats[1]);
1428                 return;
1429         }
1430         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1431         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1432         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1433                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1434                 return;
1435         }
1436         printf(" + %d%% of %d pkts + %d%% of others]\n",
1437                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1438 }
1439 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1440
1441 static void
1442 fwd_stream_stats_display(streamid_t stream_id)
1443 {
1444         struct fwd_stream *fs;
1445         static const char *fwd_top_stats_border = "-------";
1446
1447         fs = fwd_streams[stream_id];
1448         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1449             (fs->fwd_dropped == 0))
1450                 return;
1451         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1452                "TX Port=%2d/Queue=%2d %s\n",
1453                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1454                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1455         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1456                " TX-dropped: %-14"PRIu64,
1457                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1458
1459         /* if checksum mode */
1460         if (cur_fwd_eng == &csum_fwd_engine) {
1461                 printf("  RX- bad IP checksum: %-14"PRIu64
1462                        "  Rx- bad L4 checksum: %-14"PRIu64
1463                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1464                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1465                         fs->rx_bad_outer_l4_csum);
1466         } else {
1467                 printf("\n");
1468         }
1469
1470 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1471         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1472         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1473 #endif
1474 }
1475
1476 void
1477 fwd_stats_display(void)
1478 {
1479         static const char *fwd_stats_border = "----------------------";
1480         static const char *acc_stats_border = "+++++++++++++++";
1481         struct {
1482                 struct fwd_stream *rx_stream;
1483                 struct fwd_stream *tx_stream;
1484                 uint64_t tx_dropped;
1485                 uint64_t rx_bad_ip_csum;
1486                 uint64_t rx_bad_l4_csum;
1487                 uint64_t rx_bad_outer_l4_csum;
1488         } ports_stats[RTE_MAX_ETHPORTS];
1489         uint64_t total_rx_dropped = 0;
1490         uint64_t total_tx_dropped = 0;
1491         uint64_t total_rx_nombuf = 0;
1492         struct rte_eth_stats stats;
1493 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1494         uint64_t fwd_cycles = 0;
1495 #endif
1496         uint64_t total_recv = 0;
1497         uint64_t total_xmit = 0;
1498         struct rte_port *port;
1499         streamid_t sm_id;
1500         portid_t pt_id;
1501         int i;
1502
1503         memset(ports_stats, 0, sizeof(ports_stats));
1504
1505         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1506                 struct fwd_stream *fs = fwd_streams[sm_id];
1507
1508                 if (cur_fwd_config.nb_fwd_streams >
1509                     cur_fwd_config.nb_fwd_ports) {
1510                         fwd_stream_stats_display(sm_id);
1511                 } else {
1512                         ports_stats[fs->tx_port].tx_stream = fs;
1513                         ports_stats[fs->rx_port].rx_stream = fs;
1514                 }
1515
1516                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1517
1518                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1519                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1520                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1521                                 fs->rx_bad_outer_l4_csum;
1522
1523 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1524                 fwd_cycles += fs->core_cycles;
1525 #endif
1526         }
1527         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1528                 uint8_t j;
1529
1530                 pt_id = fwd_ports_ids[i];
1531                 port = &ports[pt_id];
1532
1533                 rte_eth_stats_get(pt_id, &stats);
1534                 stats.ipackets -= port->stats.ipackets;
1535                 stats.opackets -= port->stats.opackets;
1536                 stats.ibytes -= port->stats.ibytes;
1537                 stats.obytes -= port->stats.obytes;
1538                 stats.imissed -= port->stats.imissed;
1539                 stats.oerrors -= port->stats.oerrors;
1540                 stats.rx_nombuf -= port->stats.rx_nombuf;
1541
1542                 total_recv += stats.ipackets;
1543                 total_xmit += stats.opackets;
1544                 total_rx_dropped += stats.imissed;
1545                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1546                 total_tx_dropped += stats.oerrors;
1547                 total_rx_nombuf  += stats.rx_nombuf;
1548
1549                 printf("\n  %s Forward statistics for port %-2d %s\n",
1550                        fwd_stats_border, pt_id, fwd_stats_border);
1551
1552                 if (!port->rx_queue_stats_mapping_enabled &&
1553                     !port->tx_queue_stats_mapping_enabled) {
1554                         printf("  RX-packets: %-14"PRIu64
1555                                " RX-dropped: %-14"PRIu64
1556                                "RX-total: %-"PRIu64"\n",
1557                                stats.ipackets, stats.imissed,
1558                                stats.ipackets + stats.imissed);
1559
1560                         if (cur_fwd_eng == &csum_fwd_engine)
1561                                 printf("  Bad-ipcsum: %-14"PRIu64
1562                                        " Bad-l4csum: %-14"PRIu64
1563                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1564                                        ports_stats[pt_id].rx_bad_ip_csum,
1565                                        ports_stats[pt_id].rx_bad_l4_csum,
1566                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1567                         if (stats.ierrors + stats.rx_nombuf > 0) {
1568                                 printf("  RX-error: %-"PRIu64"\n",
1569                                        stats.ierrors);
1570                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1571                                        stats.rx_nombuf);
1572                         }
1573
1574                         printf("  TX-packets: %-14"PRIu64
1575                                " TX-dropped: %-14"PRIu64
1576                                "TX-total: %-"PRIu64"\n",
1577                                stats.opackets, ports_stats[pt_id].tx_dropped,
1578                                stats.opackets + ports_stats[pt_id].tx_dropped);
1579                 } else {
1580                         printf("  RX-packets:             %14"PRIu64
1581                                "    RX-dropped:%14"PRIu64
1582                                "    RX-total:%14"PRIu64"\n",
1583                                stats.ipackets, stats.imissed,
1584                                stats.ipackets + stats.imissed);
1585
1586                         if (cur_fwd_eng == &csum_fwd_engine)
1587                                 printf("  Bad-ipcsum:%14"PRIu64
1588                                        "    Bad-l4csum:%14"PRIu64
1589                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1590                                        ports_stats[pt_id].rx_bad_ip_csum,
1591                                        ports_stats[pt_id].rx_bad_l4_csum,
1592                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1593                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1594                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1595                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1596                                        stats.rx_nombuf);
1597                         }
1598
1599                         printf("  TX-packets:             %14"PRIu64
1600                                "    TX-dropped:%14"PRIu64
1601                                "    TX-total:%14"PRIu64"\n",
1602                                stats.opackets, ports_stats[pt_id].tx_dropped,
1603                                stats.opackets + ports_stats[pt_id].tx_dropped);
1604                 }
1605
1606 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1607                 if (ports_stats[pt_id].rx_stream)
1608                         pkt_burst_stats_display("RX",
1609                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1610                 if (ports_stats[pt_id].tx_stream)
1611                         pkt_burst_stats_display("TX",
1612                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1613 #endif
1614
1615                 if (port->rx_queue_stats_mapping_enabled) {
1616                         printf("\n");
1617                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1618                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1619                                        "     RX-errors:%14"PRIu64
1620                                        "    RX-bytes:%14"PRIu64"\n",
1621                                        j, stats.q_ipackets[j],
1622                                        stats.q_errors[j], stats.q_ibytes[j]);
1623                         }
1624                         printf("\n");
1625                 }
1626                 if (port->tx_queue_stats_mapping_enabled) {
1627                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1628                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1629                                        "                                 TX-bytes:%14"
1630                                        PRIu64"\n",
1631                                        j, stats.q_opackets[j],
1632                                        stats.q_obytes[j]);
1633                         }
1634                 }
1635
1636                 printf("  %s--------------------------------%s\n",
1637                        fwd_stats_border, fwd_stats_border);
1638         }
1639
1640         printf("\n  %s Accumulated forward statistics for all ports"
1641                "%s\n",
1642                acc_stats_border, acc_stats_border);
1643         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1644                "%-"PRIu64"\n"
1645                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1646                "%-"PRIu64"\n",
1647                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1648                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1649         if (total_rx_nombuf > 0)
1650                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1651         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1652                "%s\n",
1653                acc_stats_border, acc_stats_border);
1654 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1655         if (total_recv > 0)
1656                 printf("\n  CPU cycles/packet=%u (total cycles="
1657                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1658                        (unsigned int)(fwd_cycles / total_recv),
1659                        fwd_cycles, total_recv);
1660 #endif
1661 }
1662
1663 void
1664 fwd_stats_reset(void)
1665 {
1666         streamid_t sm_id;
1667         portid_t pt_id;
1668         int i;
1669
1670         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1671                 pt_id = fwd_ports_ids[i];
1672                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1673         }
1674         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1675                 struct fwd_stream *fs = fwd_streams[sm_id];
1676
1677                 fs->rx_packets = 0;
1678                 fs->tx_packets = 0;
1679                 fs->fwd_dropped = 0;
1680                 fs->rx_bad_ip_csum = 0;
1681                 fs->rx_bad_l4_csum = 0;
1682                 fs->rx_bad_outer_l4_csum = 0;
1683
1684 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1685                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1686                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1687 #endif
1688 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1689                 fs->core_cycles = 0;
1690 #endif
1691         }
1692 }
1693
1694 static void
1695 flush_fwd_rx_queues(void)
1696 {
1697         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1698         portid_t  rxp;
1699         portid_t port_id;
1700         queueid_t rxq;
1701         uint16_t  nb_rx;
1702         uint16_t  i;
1703         uint8_t   j;
1704         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1705         uint64_t timer_period;
1706
1707         /* convert to number of cycles */
1708         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1709
1710         for (j = 0; j < 2; j++) {
1711                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1712                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1713                                 port_id = fwd_ports_ids[rxp];
1714                                 /**
1715                                 * testpmd can stuck in the below do while loop
1716                                 * if rte_eth_rx_burst() always returns nonzero
1717                                 * packets. So timer is added to exit this loop
1718                                 * after 1sec timer expiry.
1719                                 */
1720                                 prev_tsc = rte_rdtsc();
1721                                 do {
1722                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1723                                                 pkts_burst, MAX_PKT_BURST);
1724                                         for (i = 0; i < nb_rx; i++)
1725                                                 rte_pktmbuf_free(pkts_burst[i]);
1726
1727                                         cur_tsc = rte_rdtsc();
1728                                         diff_tsc = cur_tsc - prev_tsc;
1729                                         timer_tsc += diff_tsc;
1730                                 } while ((nb_rx > 0) &&
1731                                         (timer_tsc < timer_period));
1732                                 timer_tsc = 0;
1733                         }
1734                 }
1735                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1736         }
1737 }
1738
1739 static void
1740 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1741 {
1742         struct fwd_stream **fsm;
1743         streamid_t nb_fs;
1744         streamid_t sm_id;
1745 #ifdef RTE_LIBRTE_BITRATE
1746         uint64_t tics_per_1sec;
1747         uint64_t tics_datum;
1748         uint64_t tics_current;
1749         uint16_t i, cnt_ports;
1750
1751         cnt_ports = nb_ports;
1752         tics_datum = rte_rdtsc();
1753         tics_per_1sec = rte_get_timer_hz();
1754 #endif
1755         fsm = &fwd_streams[fc->stream_idx];
1756         nb_fs = fc->stream_nb;
1757         do {
1758                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1759                         (*pkt_fwd)(fsm[sm_id]);
1760 #ifdef RTE_LIBRTE_BITRATE
1761                 if (bitrate_enabled != 0 &&
1762                                 bitrate_lcore_id == rte_lcore_id()) {
1763                         tics_current = rte_rdtsc();
1764                         if (tics_current - tics_datum >= tics_per_1sec) {
1765                                 /* Periodic bitrate calculation */
1766                                 for (i = 0; i < cnt_ports; i++)
1767                                         rte_stats_bitrate_calc(bitrate_data,
1768                                                 ports_ids[i]);
1769                                 tics_datum = tics_current;
1770                         }
1771                 }
1772 #endif
1773 #ifdef RTE_LIBRTE_LATENCY_STATS
1774                 if (latencystats_enabled != 0 &&
1775                                 latencystats_lcore_id == rte_lcore_id())
1776                         rte_latencystats_update();
1777 #endif
1778
1779         } while (! fc->stopped);
1780 }
1781
1782 static int
1783 start_pkt_forward_on_core(void *fwd_arg)
1784 {
1785         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1786                              cur_fwd_config.fwd_eng->packet_fwd);
1787         return 0;
1788 }
1789
1790 /*
1791  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1792  * Used to start communication flows in network loopback test configurations.
1793  */
1794 static int
1795 run_one_txonly_burst_on_core(void *fwd_arg)
1796 {
1797         struct fwd_lcore *fwd_lc;
1798         struct fwd_lcore tmp_lcore;
1799
1800         fwd_lc = (struct fwd_lcore *) fwd_arg;
1801         tmp_lcore = *fwd_lc;
1802         tmp_lcore.stopped = 1;
1803         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1804         return 0;
1805 }
1806
1807 /*
1808  * Launch packet forwarding:
1809  *     - Setup per-port forwarding context.
1810  *     - launch logical cores with their forwarding configuration.
1811  */
1812 static void
1813 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1814 {
1815         port_fwd_begin_t port_fwd_begin;
1816         unsigned int i;
1817         unsigned int lc_id;
1818         int diag;
1819
1820         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1821         if (port_fwd_begin != NULL) {
1822                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1823                         (*port_fwd_begin)(fwd_ports_ids[i]);
1824         }
1825         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1826                 lc_id = fwd_lcores_cpuids[i];
1827                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1828                         fwd_lcores[i]->stopped = 0;
1829                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1830                                                      fwd_lcores[i], lc_id);
1831                         if (diag != 0)
1832                                 printf("launch lcore %u failed - diag=%d\n",
1833                                        lc_id, diag);
1834                 }
1835         }
1836 }
1837
1838 /*
1839  * Launch packet forwarding configuration.
1840  */
1841 void
1842 start_packet_forwarding(int with_tx_first)
1843 {
1844         port_fwd_begin_t port_fwd_begin;
1845         port_fwd_end_t  port_fwd_end;
1846         struct rte_port *port;
1847         unsigned int i;
1848         portid_t   pt_id;
1849
1850         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1851                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1852
1853         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1854                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1855
1856         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1857                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1858                 (!nb_rxq || !nb_txq))
1859                 rte_exit(EXIT_FAILURE,
1860                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1861                         cur_fwd_eng->fwd_mode_name);
1862
1863         if (all_ports_started() == 0) {
1864                 printf("Not all ports were started\n");
1865                 return;
1866         }
1867         if (test_done == 0) {
1868                 printf("Packet forwarding already started\n");
1869                 return;
1870         }
1871
1872
1873         if(dcb_test) {
1874                 for (i = 0; i < nb_fwd_ports; i++) {
1875                         pt_id = fwd_ports_ids[i];
1876                         port = &ports[pt_id];
1877                         if (!port->dcb_flag) {
1878                                 printf("In DCB mode, all forwarding ports must "
1879                                        "be configured in this mode.\n");
1880                                 return;
1881                         }
1882                 }
1883                 if (nb_fwd_lcores == 1) {
1884                         printf("In DCB mode,the nb forwarding cores "
1885                                "should be larger than 1.\n");
1886                         return;
1887                 }
1888         }
1889         test_done = 0;
1890
1891         fwd_config_setup();
1892
1893         if(!no_flush_rx)
1894                 flush_fwd_rx_queues();
1895
1896         pkt_fwd_config_display(&cur_fwd_config);
1897         rxtx_config_display();
1898
1899         fwd_stats_reset();
1900         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1901                 pt_id = fwd_ports_ids[i];
1902                 port = &ports[pt_id];
1903                 map_port_queue_stats_mapping_registers(pt_id, port);
1904         }
1905         if (with_tx_first) {
1906                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1907                 if (port_fwd_begin != NULL) {
1908                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1909                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1910                 }
1911                 while (with_tx_first--) {
1912                         launch_packet_forwarding(
1913                                         run_one_txonly_burst_on_core);
1914                         rte_eal_mp_wait_lcore();
1915                 }
1916                 port_fwd_end = tx_only_engine.port_fwd_end;
1917                 if (port_fwd_end != NULL) {
1918                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1919                                 (*port_fwd_end)(fwd_ports_ids[i]);
1920                 }
1921         }
1922         launch_packet_forwarding(start_pkt_forward_on_core);
1923 }
1924
1925 void
1926 stop_packet_forwarding(void)
1927 {
1928         port_fwd_end_t port_fwd_end;
1929         lcoreid_t lc_id;
1930         portid_t pt_id;
1931         int i;
1932
1933         if (test_done) {
1934                 printf("Packet forwarding not started\n");
1935                 return;
1936         }
1937         printf("Telling cores to stop...");
1938         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1939                 fwd_lcores[lc_id]->stopped = 1;
1940         printf("\nWaiting for lcores to finish...\n");
1941         rte_eal_mp_wait_lcore();
1942         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1943         if (port_fwd_end != NULL) {
1944                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1945                         pt_id = fwd_ports_ids[i];
1946                         (*port_fwd_end)(pt_id);
1947                 }
1948         }
1949
1950         fwd_stats_display();
1951
1952         printf("\nDone.\n");
1953         test_done = 1;
1954 }
1955
1956 void
1957 dev_set_link_up(portid_t pid)
1958 {
1959         if (rte_eth_dev_set_link_up(pid) < 0)
1960                 printf("\nSet link up fail.\n");
1961 }
1962
1963 void
1964 dev_set_link_down(portid_t pid)
1965 {
1966         if (rte_eth_dev_set_link_down(pid) < 0)
1967                 printf("\nSet link down fail.\n");
1968 }
1969
1970 static int
1971 all_ports_started(void)
1972 {
1973         portid_t pi;
1974         struct rte_port *port;
1975
1976         RTE_ETH_FOREACH_DEV(pi) {
1977                 port = &ports[pi];
1978                 /* Check if there is a port which is not started */
1979                 if ((port->port_status != RTE_PORT_STARTED) &&
1980                         (port->slave_flag == 0))
1981                         return 0;
1982         }
1983
1984         /* No port is not started */
1985         return 1;
1986 }
1987
1988 int
1989 port_is_stopped(portid_t port_id)
1990 {
1991         struct rte_port *port = &ports[port_id];
1992
1993         if ((port->port_status != RTE_PORT_STOPPED) &&
1994             (port->slave_flag == 0))
1995                 return 0;
1996         return 1;
1997 }
1998
1999 int
2000 all_ports_stopped(void)
2001 {
2002         portid_t pi;
2003
2004         RTE_ETH_FOREACH_DEV(pi) {
2005                 if (!port_is_stopped(pi))
2006                         return 0;
2007         }
2008
2009         return 1;
2010 }
2011
2012 int
2013 port_is_started(portid_t port_id)
2014 {
2015         if (port_id_is_invalid(port_id, ENABLED_WARN))
2016                 return 0;
2017
2018         if (ports[port_id].port_status != RTE_PORT_STARTED)
2019                 return 0;
2020
2021         return 1;
2022 }
2023
2024 int
2025 start_port(portid_t pid)
2026 {
2027         int diag, need_check_link_status = -1;
2028         portid_t pi;
2029         queueid_t qi;
2030         struct rte_port *port;
2031         struct rte_ether_addr mac_addr;
2032
2033         if (port_id_is_invalid(pid, ENABLED_WARN))
2034                 return 0;
2035
2036         if(dcb_config)
2037                 dcb_test = 1;
2038         RTE_ETH_FOREACH_DEV(pi) {
2039                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2040                         continue;
2041
2042                 need_check_link_status = 0;
2043                 port = &ports[pi];
2044                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2045                                                  RTE_PORT_HANDLING) == 0) {
2046                         printf("Port %d is now not stopped\n", pi);
2047                         continue;
2048                 }
2049
2050                 if (port->need_reconfig > 0) {
2051                         port->need_reconfig = 0;
2052
2053                         if (flow_isolate_all) {
2054                                 int ret = port_flow_isolate(pi, 1);
2055                                 if (ret) {
2056                                         printf("Failed to apply isolated"
2057                                                " mode on port %d\n", pi);
2058                                         return -1;
2059                                 }
2060                         }
2061                         configure_rxtx_dump_callbacks(0);
2062                         printf("Configuring Port %d (socket %u)\n", pi,
2063                                         port->socket_id);
2064                         /* configure port */
2065                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
2066                                                 &(port->dev_conf));
2067                         if (diag != 0) {
2068                                 if (rte_atomic16_cmpset(&(port->port_status),
2069                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2070                                         printf("Port %d can not be set back "
2071                                                         "to stopped\n", pi);
2072                                 printf("Fail to configure port %d\n", pi);
2073                                 /* try to reconfigure port next time */
2074                                 port->need_reconfig = 1;
2075                                 return -1;
2076                         }
2077                 }
2078                 if (port->need_reconfig_queues > 0) {
2079                         port->need_reconfig_queues = 0;
2080                         /* setup tx queues */
2081                         for (qi = 0; qi < nb_txq; qi++) {
2082                                 if ((numa_support) &&
2083                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2084                                         diag = rte_eth_tx_queue_setup(pi, qi,
2085                                                 port->nb_tx_desc[qi],
2086                                                 txring_numa[pi],
2087                                                 &(port->tx_conf[qi]));
2088                                 else
2089                                         diag = rte_eth_tx_queue_setup(pi, qi,
2090                                                 port->nb_tx_desc[qi],
2091                                                 port->socket_id,
2092                                                 &(port->tx_conf[qi]));
2093
2094                                 if (diag == 0)
2095                                         continue;
2096
2097                                 /* Fail to setup tx queue, return */
2098                                 if (rte_atomic16_cmpset(&(port->port_status),
2099                                                         RTE_PORT_HANDLING,
2100                                                         RTE_PORT_STOPPED) == 0)
2101                                         printf("Port %d can not be set back "
2102                                                         "to stopped\n", pi);
2103                                 printf("Fail to configure port %d tx queues\n",
2104                                        pi);
2105                                 /* try to reconfigure queues next time */
2106                                 port->need_reconfig_queues = 1;
2107                                 return -1;
2108                         }
2109                         for (qi = 0; qi < nb_rxq; qi++) {
2110                                 /* setup rx queues */
2111                                 if ((numa_support) &&
2112                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2113                                         struct rte_mempool * mp =
2114                                                 mbuf_pool_find(rxring_numa[pi]);
2115                                         if (mp == NULL) {
2116                                                 printf("Failed to setup RX queue:"
2117                                                         "No mempool allocation"
2118                                                         " on the socket %d\n",
2119                                                         rxring_numa[pi]);
2120                                                 return -1;
2121                                         }
2122
2123                                         diag = rte_eth_rx_queue_setup(pi, qi,
2124                                              port->nb_rx_desc[qi],
2125                                              rxring_numa[pi],
2126                                              &(port->rx_conf[qi]),
2127                                              mp);
2128                                 } else {
2129                                         struct rte_mempool *mp =
2130                                                 mbuf_pool_find(port->socket_id);
2131                                         if (mp == NULL) {
2132                                                 printf("Failed to setup RX queue:"
2133                                                         "No mempool allocation"
2134                                                         " on the socket %d\n",
2135                                                         port->socket_id);
2136                                                 return -1;
2137                                         }
2138                                         diag = rte_eth_rx_queue_setup(pi, qi,
2139                                              port->nb_rx_desc[qi],
2140                                              port->socket_id,
2141                                              &(port->rx_conf[qi]),
2142                                              mp);
2143                                 }
2144                                 if (diag == 0)
2145                                         continue;
2146
2147                                 /* Fail to setup rx queue, return */
2148                                 if (rte_atomic16_cmpset(&(port->port_status),
2149                                                         RTE_PORT_HANDLING,
2150                                                         RTE_PORT_STOPPED) == 0)
2151                                         printf("Port %d can not be set back "
2152                                                         "to stopped\n", pi);
2153                                 printf("Fail to configure port %d rx queues\n",
2154                                        pi);
2155                                 /* try to reconfigure queues next time */
2156                                 port->need_reconfig_queues = 1;
2157                                 return -1;
2158                         }
2159                 }
2160                 configure_rxtx_dump_callbacks(verbose_level);
2161                 /* start port */
2162                 if (rte_eth_dev_start(pi) < 0) {
2163                         printf("Fail to start port %d\n", pi);
2164
2165                         /* Fail to setup rx queue, return */
2166                         if (rte_atomic16_cmpset(&(port->port_status),
2167                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2168                                 printf("Port %d can not be set back to "
2169                                                         "stopped\n", pi);
2170                         continue;
2171                 }
2172
2173                 if (rte_atomic16_cmpset(&(port->port_status),
2174                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2175                         printf("Port %d can not be set into started\n", pi);
2176
2177                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2178                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2179                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2180                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2181                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2182
2183                 /* at least one port started, need checking link status */
2184                 need_check_link_status = 1;
2185         }
2186
2187         if (need_check_link_status == 1 && !no_link_check)
2188                 check_all_ports_link_status(RTE_PORT_ALL);
2189         else if (need_check_link_status == 0)
2190                 printf("Please stop the ports first\n");
2191
2192         printf("Done\n");
2193         return 0;
2194 }
2195
2196 void
2197 stop_port(portid_t pid)
2198 {
2199         portid_t pi;
2200         struct rte_port *port;
2201         int need_check_link_status = 0;
2202
2203         if (dcb_test) {
2204                 dcb_test = 0;
2205                 dcb_config = 0;
2206         }
2207
2208         if (port_id_is_invalid(pid, ENABLED_WARN))
2209                 return;
2210
2211         printf("Stopping ports...\n");
2212
2213         RTE_ETH_FOREACH_DEV(pi) {
2214                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2215                         continue;
2216
2217                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2218                         printf("Please remove port %d from forwarding configuration.\n", pi);
2219                         continue;
2220                 }
2221
2222                 if (port_is_bonding_slave(pi)) {
2223                         printf("Please remove port %d from bonded device.\n", pi);
2224                         continue;
2225                 }
2226
2227                 port = &ports[pi];
2228                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2229                                                 RTE_PORT_HANDLING) == 0)
2230                         continue;
2231
2232                 rte_eth_dev_stop(pi);
2233
2234                 if (rte_atomic16_cmpset(&(port->port_status),
2235                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2236                         printf("Port %d can not be set into stopped\n", pi);
2237                 need_check_link_status = 1;
2238         }
2239         if (need_check_link_status && !no_link_check)
2240                 check_all_ports_link_status(RTE_PORT_ALL);
2241
2242         printf("Done\n");
2243 }
2244
2245 static void
2246 remove_invalid_ports_in(portid_t *array, portid_t *total)
2247 {
2248         portid_t i;
2249         portid_t new_total = 0;
2250
2251         for (i = 0; i < *total; i++)
2252                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2253                         array[new_total] = array[i];
2254                         new_total++;
2255                 }
2256         *total = new_total;
2257 }
2258
2259 static void
2260 remove_invalid_ports(void)
2261 {
2262         remove_invalid_ports_in(ports_ids, &nb_ports);
2263         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2264         nb_cfg_ports = nb_fwd_ports;
2265 }
2266
2267 void
2268 close_port(portid_t pid)
2269 {
2270         portid_t pi;
2271         struct rte_port *port;
2272
2273         if (port_id_is_invalid(pid, ENABLED_WARN))
2274                 return;
2275
2276         printf("Closing ports...\n");
2277
2278         RTE_ETH_FOREACH_DEV(pi) {
2279                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2280                         continue;
2281
2282                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2283                         printf("Please remove port %d from forwarding configuration.\n", pi);
2284                         continue;
2285                 }
2286
2287                 if (port_is_bonding_slave(pi)) {
2288                         printf("Please remove port %d from bonded device.\n", pi);
2289                         continue;
2290                 }
2291
2292                 port = &ports[pi];
2293                 if (rte_atomic16_cmpset(&(port->port_status),
2294                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2295                         printf("Port %d is already closed\n", pi);
2296                         continue;
2297                 }
2298
2299                 if (rte_atomic16_cmpset(&(port->port_status),
2300                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2301                         printf("Port %d is now not stopped\n", pi);
2302                         continue;
2303                 }
2304
2305                 if (port->flow_list)
2306                         port_flow_flush(pi);
2307                 rte_eth_dev_close(pi);
2308
2309                 remove_invalid_ports();
2310
2311                 if (rte_atomic16_cmpset(&(port->port_status),
2312                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2313                         printf("Port %d cannot be set to closed\n", pi);
2314         }
2315
2316         printf("Done\n");
2317 }
2318
2319 void
2320 reset_port(portid_t pid)
2321 {
2322         int diag;
2323         portid_t pi;
2324         struct rte_port *port;
2325
2326         if (port_id_is_invalid(pid, ENABLED_WARN))
2327                 return;
2328
2329         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2330                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2331                 printf("Can not reset port(s), please stop port(s) first.\n");
2332                 return;
2333         }
2334
2335         printf("Resetting ports...\n");
2336
2337         RTE_ETH_FOREACH_DEV(pi) {
2338                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2339                         continue;
2340
2341                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2342                         printf("Please remove port %d from forwarding "
2343                                "configuration.\n", pi);
2344                         continue;
2345                 }
2346
2347                 if (port_is_bonding_slave(pi)) {
2348                         printf("Please remove port %d from bonded device.\n",
2349                                pi);
2350                         continue;
2351                 }
2352
2353                 diag = rte_eth_dev_reset(pi);
2354                 if (diag == 0) {
2355                         port = &ports[pi];
2356                         port->need_reconfig = 1;
2357                         port->need_reconfig_queues = 1;
2358                 } else {
2359                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2360                 }
2361         }
2362
2363         printf("Done\n");
2364 }
2365
2366 void
2367 attach_port(char *identifier)
2368 {
2369         portid_t pi;
2370         struct rte_dev_iterator iterator;
2371
2372         printf("Attaching a new port...\n");
2373
2374         if (identifier == NULL) {
2375                 printf("Invalid parameters are specified\n");
2376                 return;
2377         }
2378
2379         if (rte_dev_probe(identifier) < 0) {
2380                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2381                 return;
2382         }
2383
2384         /* first attach mode: event */
2385         if (setup_on_probe_event) {
2386                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2387                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2388                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2389                                         ports[pi].need_setup != 0)
2390                                 setup_attached_port(pi);
2391                 return;
2392         }
2393
2394         /* second attach mode: iterator */
2395         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2396                 /* setup ports matching the devargs used for probing */
2397                 if (port_is_forwarding(pi))
2398                         continue; /* port was already attached before */
2399                 setup_attached_port(pi);
2400         }
2401 }
2402
2403 static void
2404 setup_attached_port(portid_t pi)
2405 {
2406         unsigned int socket_id;
2407         int ret;
2408
2409         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2410         /* if socket_id is invalid, set to the first available socket. */
2411         if (check_socket_id(socket_id) < 0)
2412                 socket_id = socket_ids[0];
2413         reconfig(pi, socket_id);
2414         ret = rte_eth_promiscuous_enable(pi);
2415         if (ret != 0)
2416                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2417                         pi, rte_strerror(-ret));
2418
2419         ports_ids[nb_ports++] = pi;
2420         fwd_ports_ids[nb_fwd_ports++] = pi;
2421         nb_cfg_ports = nb_fwd_ports;
2422         ports[pi].need_setup = 0;
2423         ports[pi].port_status = RTE_PORT_STOPPED;
2424
2425         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2426         printf("Done\n");
2427 }
2428
2429 void
2430 detach_port_device(portid_t port_id)
2431 {
2432         struct rte_device *dev;
2433         portid_t sibling;
2434
2435         printf("Removing a device...\n");
2436
2437         dev = rte_eth_devices[port_id].device;
2438         if (dev == NULL) {
2439                 printf("Device already removed\n");
2440                 return;
2441         }
2442
2443         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2444                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2445                         printf("Port not stopped\n");
2446                         return;
2447                 }
2448                 printf("Port was not closed\n");
2449                 if (ports[port_id].flow_list)
2450                         port_flow_flush(port_id);
2451         }
2452
2453         if (rte_dev_remove(dev) < 0) {
2454                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2455                 return;
2456         }
2457         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2458                 /* reset mapping between old ports and removed device */
2459                 rte_eth_devices[sibling].device = NULL;
2460                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2461                         /* sibling ports are forced to be closed */
2462                         ports[sibling].port_status = RTE_PORT_CLOSED;
2463                         printf("Port %u is closed\n", sibling);
2464                 }
2465         }
2466
2467         remove_invalid_ports();
2468
2469         printf("Device of port %u is detached\n", port_id);
2470         printf("Now total ports is %d\n", nb_ports);
2471         printf("Done\n");
2472         return;
2473 }
2474
2475 void
2476 detach_device(char *identifier)
2477 {
2478         struct rte_dev_iterator iterator;
2479         struct rte_devargs da;
2480         portid_t port_id;
2481
2482         printf("Removing a device...\n");
2483
2484         memset(&da, 0, sizeof(da));
2485         if (rte_devargs_parsef(&da, "%s", identifier)) {
2486                 printf("cannot parse identifier\n");
2487                 if (da.args)
2488                         free(da.args);
2489                 return;
2490         }
2491
2492         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2493                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2494                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2495                                 printf("Port %u not stopped\n", port_id);
2496                                 return;
2497                         }
2498
2499                         /* sibling ports are forced to be closed */
2500                         if (ports[port_id].flow_list)
2501                                 port_flow_flush(port_id);
2502                         ports[port_id].port_status = RTE_PORT_CLOSED;
2503                         printf("Port %u is now closed\n", port_id);
2504                 }
2505         }
2506
2507         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2508                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2509                             da.name, da.bus->name);
2510                 return;
2511         }
2512
2513         remove_invalid_ports();
2514
2515         printf("Device %s is detached\n", identifier);
2516         printf("Now total ports is %d\n", nb_ports);
2517         printf("Done\n");
2518 }
2519
2520 void
2521 pmd_test_exit(void)
2522 {
2523         portid_t pt_id;
2524         int ret;
2525         int i;
2526
2527         if (test_done == 0)
2528                 stop_packet_forwarding();
2529
2530         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2531                 if (mempools[i]) {
2532                         if (mp_alloc_type == MP_ALLOC_ANON)
2533                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2534                                                      NULL);
2535                 }
2536         }
2537         if (ports != NULL) {
2538                 no_link_check = 1;
2539                 RTE_ETH_FOREACH_DEV(pt_id) {
2540                         printf("\nStopping port %d...\n", pt_id);
2541                         fflush(stdout);
2542                         stop_port(pt_id);
2543                 }
2544                 RTE_ETH_FOREACH_DEV(pt_id) {
2545                         printf("\nShutting down port %d...\n", pt_id);
2546                         fflush(stdout);
2547                         close_port(pt_id);
2548                 }
2549         }
2550
2551         if (hot_plug) {
2552                 ret = rte_dev_event_monitor_stop();
2553                 if (ret) {
2554                         RTE_LOG(ERR, EAL,
2555                                 "fail to stop device event monitor.");
2556                         return;
2557                 }
2558
2559                 ret = rte_dev_event_callback_unregister(NULL,
2560                         dev_event_callback, NULL);
2561                 if (ret < 0) {
2562                         RTE_LOG(ERR, EAL,
2563                                 "fail to unregister device event callback.\n");
2564                         return;
2565                 }
2566
2567                 ret = rte_dev_hotplug_handle_disable();
2568                 if (ret) {
2569                         RTE_LOG(ERR, EAL,
2570                                 "fail to disable hotplug handling.\n");
2571                         return;
2572                 }
2573         }
2574         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2575                 if (mempools[i])
2576                         rte_mempool_free(mempools[i]);
2577         }
2578
2579         printf("\nBye...\n");
2580 }
2581
2582 typedef void (*cmd_func_t)(void);
2583 struct pmd_test_command {
2584         const char *cmd_name;
2585         cmd_func_t cmd_func;
2586 };
2587
2588 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2589
2590 /* Check the link status of all ports in up to 9s, and print them finally */
2591 static void
2592 check_all_ports_link_status(uint32_t port_mask)
2593 {
2594 #define CHECK_INTERVAL 100 /* 100ms */
2595 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2596         portid_t portid;
2597         uint8_t count, all_ports_up, print_flag = 0;
2598         struct rte_eth_link link;
2599         int ret;
2600
2601         printf("Checking link statuses...\n");
2602         fflush(stdout);
2603         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2604                 all_ports_up = 1;
2605                 RTE_ETH_FOREACH_DEV(portid) {
2606                         if ((port_mask & (1 << portid)) == 0)
2607                                 continue;
2608                         memset(&link, 0, sizeof(link));
2609                         ret = rte_eth_link_get_nowait(portid, &link);
2610                         if (ret < 0) {
2611                                 all_ports_up = 0;
2612                                 if (print_flag == 1)
2613                                         printf("Port %u link get failed: %s\n",
2614                                                 portid, rte_strerror(-ret));
2615                                 continue;
2616                         }
2617                         /* print link status if flag set */
2618                         if (print_flag == 1) {
2619                                 if (link.link_status)
2620                                         printf(
2621                                         "Port%d Link Up. speed %u Mbps- %s\n",
2622                                         portid, link.link_speed,
2623                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2624                                         ("full-duplex") : ("half-duplex\n"));
2625                                 else
2626                                         printf("Port %d Link Down\n", portid);
2627                                 continue;
2628                         }
2629                         /* clear all_ports_up flag if any link down */
2630                         if (link.link_status == ETH_LINK_DOWN) {
2631                                 all_ports_up = 0;
2632                                 break;
2633                         }
2634                 }
2635                 /* after finally printing all link status, get out */
2636                 if (print_flag == 1)
2637                         break;
2638
2639                 if (all_ports_up == 0) {
2640                         fflush(stdout);
2641                         rte_delay_ms(CHECK_INTERVAL);
2642                 }
2643
2644                 /* set the print_flag if all ports up or timeout */
2645                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2646                         print_flag = 1;
2647                 }
2648
2649                 if (lsc_interrupt)
2650                         break;
2651         }
2652 }
2653
2654 /*
2655  * This callback is for remove a port for a device. It has limitation because
2656  * it is not for multiple port removal for a device.
2657  * TODO: the device detach invoke will plan to be removed from user side to
2658  * eal. And convert all PMDs to free port resources on ether device closing.
2659  */
2660 static void
2661 rmv_port_callback(void *arg)
2662 {
2663         int need_to_start = 0;
2664         int org_no_link_check = no_link_check;
2665         portid_t port_id = (intptr_t)arg;
2666
2667         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2668
2669         if (!test_done && port_is_forwarding(port_id)) {
2670                 need_to_start = 1;
2671                 stop_packet_forwarding();
2672         }
2673         no_link_check = 1;
2674         stop_port(port_id);
2675         no_link_check = org_no_link_check;
2676         close_port(port_id);
2677         detach_port_device(port_id);
2678         if (need_to_start)
2679                 start_packet_forwarding(0);
2680 }
2681
2682 /* This function is used by the interrupt thread */
2683 static int
2684 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2685                   void *ret_param)
2686 {
2687         RTE_SET_USED(param);
2688         RTE_SET_USED(ret_param);
2689
2690         if (type >= RTE_ETH_EVENT_MAX) {
2691                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2692                         port_id, __func__, type);
2693                 fflush(stderr);
2694         } else if (event_print_mask & (UINT32_C(1) << type)) {
2695                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2696                         eth_event_desc[type]);
2697                 fflush(stdout);
2698         }
2699
2700         switch (type) {
2701         case RTE_ETH_EVENT_NEW:
2702                 ports[port_id].need_setup = 1;
2703                 ports[port_id].port_status = RTE_PORT_HANDLING;
2704                 break;
2705         case RTE_ETH_EVENT_INTR_RMV:
2706                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2707                         break;
2708                 if (rte_eal_alarm_set(100000,
2709                                 rmv_port_callback, (void *)(intptr_t)port_id))
2710                         fprintf(stderr, "Could not set up deferred device removal\n");
2711                 break;
2712         default:
2713                 break;
2714         }
2715         return 0;
2716 }
2717
2718 static int
2719 register_eth_event_callback(void)
2720 {
2721         int ret;
2722         enum rte_eth_event_type event;
2723
2724         for (event = RTE_ETH_EVENT_UNKNOWN;
2725                         event < RTE_ETH_EVENT_MAX; event++) {
2726                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2727                                 event,
2728                                 eth_event_callback,
2729                                 NULL);
2730                 if (ret != 0) {
2731                         TESTPMD_LOG(ERR, "Failed to register callback for "
2732                                         "%s event\n", eth_event_desc[event]);
2733                         return -1;
2734                 }
2735         }
2736
2737         return 0;
2738 }
2739
2740 /* This function is used by the interrupt thread */
2741 static void
2742 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2743                              __rte_unused void *arg)
2744 {
2745         uint16_t port_id;
2746         int ret;
2747
2748         if (type >= RTE_DEV_EVENT_MAX) {
2749                 fprintf(stderr, "%s called upon invalid event %d\n",
2750                         __func__, type);
2751                 fflush(stderr);
2752         }
2753
2754         switch (type) {
2755         case RTE_DEV_EVENT_REMOVE:
2756                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2757                         device_name);
2758                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2759                 if (ret) {
2760                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2761                                 device_name);
2762                         return;
2763                 }
2764                 /*
2765                  * Because the user's callback is invoked in eal interrupt
2766                  * callback, the interrupt callback need to be finished before
2767                  * it can be unregistered when detaching device. So finish
2768                  * callback soon and use a deferred removal to detach device
2769                  * is need. It is a workaround, once the device detaching be
2770                  * moved into the eal in the future, the deferred removal could
2771                  * be deleted.
2772                  */
2773                 if (rte_eal_alarm_set(100000,
2774                                 rmv_port_callback, (void *)(intptr_t)port_id))
2775                         RTE_LOG(ERR, EAL,
2776                                 "Could not set up deferred device removal\n");
2777                 break;
2778         case RTE_DEV_EVENT_ADD:
2779                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2780                         device_name);
2781                 /* TODO: After finish kernel driver binding,
2782                  * begin to attach port.
2783                  */
2784                 break;
2785         default:
2786                 break;
2787         }
2788 }
2789
2790 static int
2791 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2792 {
2793         uint16_t i;
2794         int diag;
2795         uint8_t mapping_found = 0;
2796
2797         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2798                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2799                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2800                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2801                                         tx_queue_stats_mappings[i].queue_id,
2802                                         tx_queue_stats_mappings[i].stats_counter_id);
2803                         if (diag != 0)
2804                                 return diag;
2805                         mapping_found = 1;
2806                 }
2807         }
2808         if (mapping_found)
2809                 port->tx_queue_stats_mapping_enabled = 1;
2810         return 0;
2811 }
2812
2813 static int
2814 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2815 {
2816         uint16_t i;
2817         int diag;
2818         uint8_t mapping_found = 0;
2819
2820         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2821                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2822                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2823                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2824                                         rx_queue_stats_mappings[i].queue_id,
2825                                         rx_queue_stats_mappings[i].stats_counter_id);
2826                         if (diag != 0)
2827                                 return diag;
2828                         mapping_found = 1;
2829                 }
2830         }
2831         if (mapping_found)
2832                 port->rx_queue_stats_mapping_enabled = 1;
2833         return 0;
2834 }
2835
2836 static void
2837 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2838 {
2839         int diag = 0;
2840
2841         diag = set_tx_queue_stats_mapping_registers(pi, port);
2842         if (diag != 0) {
2843                 if (diag == -ENOTSUP) {
2844                         port->tx_queue_stats_mapping_enabled = 0;
2845                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2846                 }
2847                 else
2848                         rte_exit(EXIT_FAILURE,
2849                                         "set_tx_queue_stats_mapping_registers "
2850                                         "failed for port id=%d diag=%d\n",
2851                                         pi, diag);
2852         }
2853
2854         diag = set_rx_queue_stats_mapping_registers(pi, port);
2855         if (diag != 0) {
2856                 if (diag == -ENOTSUP) {
2857                         port->rx_queue_stats_mapping_enabled = 0;
2858                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2859                 }
2860                 else
2861                         rte_exit(EXIT_FAILURE,
2862                                         "set_rx_queue_stats_mapping_registers "
2863                                         "failed for port id=%d diag=%d\n",
2864                                         pi, diag);
2865         }
2866 }
2867
2868 static void
2869 rxtx_port_config(struct rte_port *port)
2870 {
2871         uint16_t qid;
2872         uint64_t offloads;
2873
2874         for (qid = 0; qid < nb_rxq; qid++) {
2875                 offloads = port->rx_conf[qid].offloads;
2876                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2877                 if (offloads != 0)
2878                         port->rx_conf[qid].offloads = offloads;
2879
2880                 /* Check if any Rx parameters have been passed */
2881                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2882                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2883
2884                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2885                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2886
2887                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2888                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2889
2890                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2891                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2892
2893                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2894                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2895
2896                 port->nb_rx_desc[qid] = nb_rxd;
2897         }
2898
2899         for (qid = 0; qid < nb_txq; qid++) {
2900                 offloads = port->tx_conf[qid].offloads;
2901                 port->tx_conf[qid] = port->dev_info.default_txconf;
2902                 if (offloads != 0)
2903                         port->tx_conf[qid].offloads = offloads;
2904
2905                 /* Check if any Tx parameters have been passed */
2906                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2907                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2908
2909                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2910                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2911
2912                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2913                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2914
2915                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2916                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2917
2918                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2919                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2920
2921                 port->nb_tx_desc[qid] = nb_txd;
2922         }
2923 }
2924
2925 void
2926 init_port_config(void)
2927 {
2928         portid_t pid;
2929         struct rte_port *port;
2930         int ret;
2931
2932         RTE_ETH_FOREACH_DEV(pid) {
2933                 port = &ports[pid];
2934                 port->dev_conf.fdir_conf = fdir_conf;
2935
2936                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
2937                 if (ret != 0)
2938                         return;
2939
2940                 if (nb_rxq > 1) {
2941                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2942                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2943                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2944                 } else {
2945                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2946                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2947                 }
2948
2949                 if (port->dcb_flag == 0) {
2950                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2951                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2952                         else
2953                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2954                 }
2955
2956                 rxtx_port_config(port);
2957
2958                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
2959                 if (ret != 0)
2960                         return;
2961
2962                 map_port_queue_stats_mapping_registers(pid, port);
2963 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2964                 rte_pmd_ixgbe_bypass_init(pid);
2965 #endif
2966
2967                 if (lsc_interrupt &&
2968                     (rte_eth_devices[pid].data->dev_flags &
2969                      RTE_ETH_DEV_INTR_LSC))
2970                         port->dev_conf.intr_conf.lsc = 1;
2971                 if (rmv_interrupt &&
2972                     (rte_eth_devices[pid].data->dev_flags &
2973                      RTE_ETH_DEV_INTR_RMV))
2974                         port->dev_conf.intr_conf.rmv = 1;
2975         }
2976 }
2977
2978 void set_port_slave_flag(portid_t slave_pid)
2979 {
2980         struct rte_port *port;
2981
2982         port = &ports[slave_pid];
2983         port->slave_flag = 1;
2984 }
2985
2986 void clear_port_slave_flag(portid_t slave_pid)
2987 {
2988         struct rte_port *port;
2989
2990         port = &ports[slave_pid];
2991         port->slave_flag = 0;
2992 }
2993
2994 uint8_t port_is_bonding_slave(portid_t slave_pid)
2995 {
2996         struct rte_port *port;
2997
2998         port = &ports[slave_pid];
2999         if ((rte_eth_devices[slave_pid].data->dev_flags &
3000             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3001                 return 1;
3002         return 0;
3003 }
3004
3005 const uint16_t vlan_tags[] = {
3006                 0,  1,  2,  3,  4,  5,  6,  7,
3007                 8,  9, 10, 11,  12, 13, 14, 15,
3008                 16, 17, 18, 19, 20, 21, 22, 23,
3009                 24, 25, 26, 27, 28, 29, 30, 31
3010 };
3011
3012 static  int
3013 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3014                  enum dcb_mode_enable dcb_mode,
3015                  enum rte_eth_nb_tcs num_tcs,
3016                  uint8_t pfc_en)
3017 {
3018         uint8_t i;
3019         int32_t rc;
3020         struct rte_eth_rss_conf rss_conf;
3021
3022         /*
3023          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3024          * given above, and the number of traffic classes available for use.
3025          */
3026         if (dcb_mode == DCB_VT_ENABLED) {
3027                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3028                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3029                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3030                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3031
3032                 /* VMDQ+DCB RX and TX configurations */
3033                 vmdq_rx_conf->enable_default_pool = 0;
3034                 vmdq_rx_conf->default_pool = 0;
3035                 vmdq_rx_conf->nb_queue_pools =
3036                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3037                 vmdq_tx_conf->nb_queue_pools =
3038                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3039
3040                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3041                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3042                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3043                         vmdq_rx_conf->pool_map[i].pools =
3044                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3045                 }
3046                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3047                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3048                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3049                 }
3050
3051                 /* set DCB mode of RX and TX of multiple queues */
3052                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
3053                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3054         } else {
3055                 struct rte_eth_dcb_rx_conf *rx_conf =
3056                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3057                 struct rte_eth_dcb_tx_conf *tx_conf =
3058                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3059
3060                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3061                 if (rc != 0)
3062                         return rc;
3063
3064                 rx_conf->nb_tcs = num_tcs;
3065                 tx_conf->nb_tcs = num_tcs;
3066
3067                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3068                         rx_conf->dcb_tc[i] = i % num_tcs;
3069                         tx_conf->dcb_tc[i] = i % num_tcs;
3070                 }
3071
3072                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
3073                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3074                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3075         }
3076
3077         if (pfc_en)
3078                 eth_conf->dcb_capability_en =
3079                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3080         else
3081                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3082
3083         return 0;
3084 }
3085
3086 int
3087 init_port_dcb_config(portid_t pid,
3088                      enum dcb_mode_enable dcb_mode,
3089                      enum rte_eth_nb_tcs num_tcs,
3090                      uint8_t pfc_en)
3091 {
3092         struct rte_eth_conf port_conf;
3093         struct rte_port *rte_port;
3094         int retval;
3095         uint16_t i;
3096
3097         rte_port = &ports[pid];
3098
3099         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3100         /* Enter DCB configuration status */
3101         dcb_config = 1;
3102
3103         port_conf.rxmode = rte_port->dev_conf.rxmode;
3104         port_conf.txmode = rte_port->dev_conf.txmode;
3105
3106         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3107         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3108         if (retval < 0)
3109                 return retval;
3110         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3111
3112         /* re-configure the device . */
3113         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3114         if (retval < 0)
3115                 return retval;
3116
3117         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3118         if (retval != 0)
3119                 return retval;
3120
3121         /* If dev_info.vmdq_pool_base is greater than 0,
3122          * the queue id of vmdq pools is started after pf queues.
3123          */
3124         if (dcb_mode == DCB_VT_ENABLED &&
3125             rte_port->dev_info.vmdq_pool_base > 0) {
3126                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3127                         " for port %d.", pid);
3128                 return -1;
3129         }
3130
3131         /* Assume the ports in testpmd have the same dcb capability
3132          * and has the same number of rxq and txq in dcb mode
3133          */
3134         if (dcb_mode == DCB_VT_ENABLED) {
3135                 if (rte_port->dev_info.max_vfs > 0) {
3136                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3137                         nb_txq = rte_port->dev_info.nb_tx_queues;
3138                 } else {
3139                         nb_rxq = rte_port->dev_info.max_rx_queues;
3140                         nb_txq = rte_port->dev_info.max_tx_queues;
3141                 }
3142         } else {
3143                 /*if vt is disabled, use all pf queues */
3144                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3145                         nb_rxq = rte_port->dev_info.max_rx_queues;
3146                         nb_txq = rte_port->dev_info.max_tx_queues;
3147                 } else {
3148                         nb_rxq = (queueid_t)num_tcs;
3149                         nb_txq = (queueid_t)num_tcs;
3150
3151                 }
3152         }
3153         rx_free_thresh = 64;
3154
3155         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3156
3157         rxtx_port_config(rte_port);
3158         /* VLAN filter */
3159         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3160         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3161                 rx_vft_set(pid, vlan_tags[i], 1);
3162
3163         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3164         if (retval != 0)
3165                 return retval;
3166
3167         map_port_queue_stats_mapping_registers(pid, rte_port);
3168
3169         rte_port->dcb_flag = 1;
3170
3171         return 0;
3172 }
3173
3174 static void
3175 init_port(void)
3176 {
3177         /* Configuration of Ethernet ports. */
3178         ports = rte_zmalloc("testpmd: ports",
3179                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3180                             RTE_CACHE_LINE_SIZE);
3181         if (ports == NULL) {
3182                 rte_exit(EXIT_FAILURE,
3183                                 "rte_zmalloc(%d struct rte_port) failed\n",
3184                                 RTE_MAX_ETHPORTS);
3185         }
3186
3187         /* Initialize ports NUMA structures */
3188         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3189         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3190         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3191 }
3192
3193 static void
3194 force_quit(void)
3195 {
3196         pmd_test_exit();
3197         prompt_exit();
3198 }
3199
3200 static void
3201 print_stats(void)
3202 {
3203         uint8_t i;
3204         const char clr[] = { 27, '[', '2', 'J', '\0' };
3205         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3206
3207         /* Clear screen and move to top left */
3208         printf("%s%s", clr, top_left);
3209
3210         printf("\nPort statistics ====================================");
3211         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3212                 nic_stats_display(fwd_ports_ids[i]);
3213
3214         fflush(stdout);
3215 }
3216
3217 static void
3218 signal_handler(int signum)
3219 {
3220         if (signum == SIGINT || signum == SIGTERM) {
3221                 printf("\nSignal %d received, preparing to exit...\n",
3222                                 signum);
3223 #ifdef RTE_LIBRTE_PDUMP
3224                 /* uninitialize packet capture framework */
3225                 rte_pdump_uninit();
3226 #endif
3227 #ifdef RTE_LIBRTE_LATENCY_STATS
3228                 if (latencystats_enabled != 0)
3229                         rte_latencystats_uninit();
3230 #endif
3231                 force_quit();
3232                 /* Set flag to indicate the force termination. */
3233                 f_quit = 1;
3234                 /* exit with the expected status */
3235                 signal(signum, SIG_DFL);
3236                 kill(getpid(), signum);
3237         }
3238 }
3239
3240 int
3241 main(int argc, char** argv)
3242 {
3243         int diag;
3244         portid_t port_id;
3245         uint16_t count;
3246         int ret;
3247
3248         signal(SIGINT, signal_handler);
3249         signal(SIGTERM, signal_handler);
3250
3251         testpmd_logtype = rte_log_register("testpmd");
3252         if (testpmd_logtype < 0)
3253                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3254         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3255
3256         diag = rte_eal_init(argc, argv);
3257         if (diag < 0)
3258                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3259                          rte_strerror(rte_errno));
3260
3261         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3262                 rte_exit(EXIT_FAILURE,
3263                          "Secondary process type not supported.\n");
3264
3265         ret = register_eth_event_callback();
3266         if (ret != 0)
3267                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3268
3269 #ifdef RTE_LIBRTE_PDUMP
3270         /* initialize packet capture framework */
3271         rte_pdump_init();
3272 #endif
3273
3274         count = 0;
3275         RTE_ETH_FOREACH_DEV(port_id) {
3276                 ports_ids[count] = port_id;
3277                 count++;
3278         }
3279         nb_ports = (portid_t) count;
3280         if (nb_ports == 0)
3281                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3282
3283         /* allocate port structures, and init them */
3284         init_port();
3285
3286         set_def_fwd_config();
3287         if (nb_lcores == 0)
3288                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3289                          "Check the core mask argument\n");
3290
3291         /* Bitrate/latency stats disabled by default */
3292 #ifdef RTE_LIBRTE_BITRATE
3293         bitrate_enabled = 0;
3294 #endif
3295 #ifdef RTE_LIBRTE_LATENCY_STATS
3296         latencystats_enabled = 0;
3297 #endif
3298
3299         /* on FreeBSD, mlockall() is disabled by default */
3300 #ifdef RTE_EXEC_ENV_FREEBSD
3301         do_mlockall = 0;
3302 #else
3303         do_mlockall = 1;
3304 #endif
3305
3306         argc -= diag;
3307         argv += diag;
3308         if (argc > 1)
3309                 launch_args_parse(argc, argv);
3310
3311         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3312                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3313                         strerror(errno));
3314         }
3315
3316         if (tx_first && interactive)
3317                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3318                                 "interactive mode.\n");
3319
3320         if (tx_first && lsc_interrupt) {
3321                 printf("Warning: lsc_interrupt needs to be off when "
3322                                 " using tx_first. Disabling.\n");
3323                 lsc_interrupt = 0;
3324         }
3325
3326         if (!nb_rxq && !nb_txq)
3327                 printf("Warning: Either rx or tx queues should be non-zero\n");
3328
3329         if (nb_rxq > 1 && nb_rxq > nb_txq)
3330                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3331                        "but nb_txq=%d will prevent to fully test it.\n",
3332                        nb_rxq, nb_txq);
3333
3334         init_config();
3335
3336         if (hot_plug) {
3337                 ret = rte_dev_hotplug_handle_enable();
3338                 if (ret) {
3339                         RTE_LOG(ERR, EAL,
3340                                 "fail to enable hotplug handling.");
3341                         return -1;
3342                 }
3343
3344                 ret = rte_dev_event_monitor_start();
3345                 if (ret) {
3346                         RTE_LOG(ERR, EAL,
3347                                 "fail to start device event monitoring.");
3348                         return -1;
3349                 }
3350
3351                 ret = rte_dev_event_callback_register(NULL,
3352                         dev_event_callback, NULL);
3353                 if (ret) {
3354                         RTE_LOG(ERR, EAL,
3355                                 "fail  to register device event callback\n");
3356                         return -1;
3357                 }
3358         }
3359
3360         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3361                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3362
3363         /* set all ports to promiscuous mode by default */
3364         RTE_ETH_FOREACH_DEV(port_id) {
3365                 ret = rte_eth_promiscuous_enable(port_id);
3366                 if (ret != 0)
3367                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3368                                 port_id, rte_strerror(-ret));
3369         }
3370
3371         /* Init metrics library */
3372         rte_metrics_init(rte_socket_id());
3373
3374 #ifdef RTE_LIBRTE_LATENCY_STATS
3375         if (latencystats_enabled != 0) {
3376                 int ret = rte_latencystats_init(1, NULL);
3377                 if (ret)
3378                         printf("Warning: latencystats init()"
3379                                 " returned error %d\n", ret);
3380                 printf("Latencystats running on lcore %d\n",
3381                         latencystats_lcore_id);
3382         }
3383 #endif
3384
3385         /* Setup bitrate stats */
3386 #ifdef RTE_LIBRTE_BITRATE
3387         if (bitrate_enabled != 0) {
3388                 bitrate_data = rte_stats_bitrate_create();
3389                 if (bitrate_data == NULL)
3390                         rte_exit(EXIT_FAILURE,
3391                                 "Could not allocate bitrate data.\n");
3392                 rte_stats_bitrate_reg(bitrate_data);
3393         }
3394 #endif
3395
3396 #ifdef RTE_LIBRTE_CMDLINE
3397         if (strlen(cmdline_filename) != 0)
3398                 cmdline_read_from_file(cmdline_filename);
3399
3400         if (interactive == 1) {
3401                 if (auto_start) {
3402                         printf("Start automatic packet forwarding\n");
3403                         start_packet_forwarding(0);
3404                 }
3405                 prompt();
3406                 pmd_test_exit();
3407         } else
3408 #endif
3409         {
3410                 char c;
3411                 int rc;
3412
3413                 f_quit = 0;
3414
3415                 printf("No commandline core given, start packet forwarding\n");
3416                 start_packet_forwarding(tx_first);
3417                 if (stats_period != 0) {
3418                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3419                         uint64_t timer_period;
3420
3421                         /* Convert to number of cycles */
3422                         timer_period = stats_period * rte_get_timer_hz();
3423
3424                         while (f_quit == 0) {
3425                                 cur_time = rte_get_timer_cycles();
3426                                 diff_time += cur_time - prev_time;
3427
3428                                 if (diff_time >= timer_period) {
3429                                         print_stats();
3430                                         /* Reset the timer */
3431                                         diff_time = 0;
3432                                 }
3433                                 /* Sleep to avoid unnecessary checks */
3434                                 prev_time = cur_time;
3435                                 sleep(1);
3436                         }
3437                 }
3438
3439                 printf("Press enter to exit\n");
3440                 rc = read(0, &c, 1);
3441                 pmd_test_exit();
3442                 if (rc < 0)
3443                         return 1;
3444         }
3445
3446         return 0;
3447 }