app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATE
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81
  82 uint16_t verbose_level = 0; /**< Silent by default. */
  83 int testpmd_logtype; /**< Log type for testpmd logs */
  84
  85 /* use master core for command line ? */
  86 uint8_t interactive = 0;
  87 uint8_t auto_start = 0;
  88 uint8_t tx_first;
  89 char cmdline_filename[PATH_MAX] = {0};
  90
  91 /*
  92  * NUMA support configuration.
  93  * When set, the NUMA support attempts to dispatch the allocation of the
  94  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  95  * probed ports among the CPU sockets 0 and 1.
  96  * Otherwise, all memory is allocated from CPU socket 0.
  97  */
  98 uint8_t numa_support = 1; /**< numa enabled by default */
  99
 100 /*
 101  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 102  * not configured.
 103  */
 104 uint8_t socket_num = UMA_NO_CONFIG;
 105
 106 /*
 107  * Select mempool allocation type:
 108  * - native: use regular DPDK memory
 109  * - anon: use regular DPDK memory to create mempool, but populate using
 110  *         anonymous memory (may not be IOVA-contiguous)
 111  * - xmem: use externally allocated hugepage memory
 112  */
 113 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 114
 115 /*
 116  * Store specified sockets on which memory pool to be used by ports
 117  * is allocated.
 118  */
 119 uint8_t port_numa[RTE_MAX_ETHPORTS];
 120
 121 /*
 122  * Store specified sockets on which RX ring to be used by ports
 123  * is allocated.
 124  */
 125 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 126
 127 /*
 128  * Store specified sockets on which TX ring to be used by ports
 129  * is allocated.
 130  */
 131 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 132
 133 /*
 134  * Record the Ethernet address of peer target ports to which packets are
 135  * forwarded.
 136  * Must be instantiated with the ethernet addresses of peer traffic generator
 137  * ports.
 138  */
 139 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 140 portid_t nb_peer_eth_addrs = 0;
 141
 142 /*
 143  * Probed Target Environment.
 144  */
 145 struct rte_port *ports;        /**< For all probed ethernet ports. */
 146 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 147 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 148 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 149
 150 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 151
 152 /*
 153  * Test Forwarding Configuration.
 154  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 155  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 156  */
 157 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 158 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 159 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 160 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 161
 162 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 163 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 164
 165 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 166 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 167
 168 /*
 169  * Forwarding engines.
 170  */
 171 struct fwd_engine * fwd_engines[] = {
 172         &io_fwd_engine,
 173         &mac_fwd_engine,
 174         &mac_swap_engine,
 175         &flow_gen_engine,
 176         &rx_only_engine,
 177         &tx_only_engine,
 178         &csum_fwd_engine,
 179         &icmp_echo_engine,
 180         &noisy_vnf_engine,
 181 #if defined RTE_LIBRTE_PMD_SOFTNIC
 182         &softnic_fwd_engine,
 183 #endif
 184 #ifdef RTE_LIBRTE_IEEE1588
 185         &ieee1588_fwd_engine,
 186 #endif
 187         NULL,
 188 };
 189
 190 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 191 uint16_t mempool_flags;
 192
 193 struct fwd_config cur_fwd_config;
 194 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 195 uint32_t retry_enabled;
 196 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 197 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 198
 199 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 200 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 201                                       * specified on command-line. */
 202 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 203
 204 /*
 205  * In container, it cannot terminate the process which running with 'stats-period'
 206  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 207  */
 208 uint8_t f_quit;
 209
 210 /*
 211  * Configuration of packet segments used by the "txonly" processing engine.
 212  */
 213 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 214 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 215         TXONLY_DEF_PACKET_LEN,
 216 };
 217 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 218
 219 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 220 /**< Split policy for packets to TX. */
 221
 222 uint8_t txonly_multi_flow;
 223 /**< Whether multiple flows are generated in TXONLY mode. */
 224
 225 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 226 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 227
 228 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 229 uint8_t dcb_config = 0;
 230
 231 /* Whether the dcb is in testing status */
 232 uint8_t dcb_test = 0;
 233
 234 /*
 235  * Configurable number of RX/TX queues.
 236  */
 237 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 238 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 239
 240 /*
 241  * Configurable number of RX/TX ring descriptors.
 242  * Defaults are supplied by drivers via ethdev.
 243  */
 244 #define RTE_TEST_RX_DESC_DEFAULT 0
 245 #define RTE_TEST_TX_DESC_DEFAULT 0
 246 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 247 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 248
 249 #define RTE_PMD_PARAM_UNSET -1
 250 /*
 251  * Configurable values of RX and TX ring threshold registers.
 252  */
 253
 254 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 255 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 256 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 257
 258 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 259 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 260 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 261
 262 /*
 263  * Configurable value of RX free threshold.
 264  */
 265 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 266
 267 /*
 268  * Configurable value of RX drop enable.
 269  */
 270 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 271
 272 /*
 273  * Configurable value of TX free threshold.
 274  */
 275 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 276
 277 /*
 278  * Configurable value of TX RS bit threshold.
 279  */
 280 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 281
 282 /*
 283  * Configurable value of buffered packets before sending.
 284  */
 285 uint16_t noisy_tx_sw_bufsz;
 286
 287 /*
 288  * Configurable value of packet buffer timeout.
 289  */
 290 uint16_t noisy_tx_sw_buf_flush_time;
 291
 292 /*
 293  * Configurable value for size of VNF internal memory area
 294  * used for simulating noisy neighbour behaviour
 295  */
 296 uint64_t noisy_lkup_mem_sz;
 297
 298 /*
 299  * Configurable value of number of random writes done in
 300  * VNF simulation memory area.
 301  */
 302 uint64_t noisy_lkup_num_writes;
 303
 304 /*
 305  * Configurable value of number of random reads done in
 306  * VNF simulation memory area.
 307  */
 308 uint64_t noisy_lkup_num_reads;
 309
 310 /*
 311  * Configurable value of number of random reads/writes done in
 312  * VNF simulation memory area.
 313  */
 314 uint64_t noisy_lkup_num_reads_writes;
 315
 316 /*
 317  * Receive Side Scaling (RSS) configuration.
 318  */
 319 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 320
 321 /*
 322  * Port topology configuration
 323  */
 324 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 325
 326 /*
 327  * Avoids to flush all the RX streams before starts forwarding.
 328  */
 329 uint8_t no_flush_rx = 0; /* flush by default */
 330
 331 /*
 332  * Flow API isolated mode.
 333  */
 334 uint8_t flow_isolate_all;
 335
 336 /*
 337  * Avoids to check link status when starting/stopping a port.
 338  */
 339 uint8_t no_link_check = 0; /* check by default */
 340
 341 /*
 342  * Don't automatically start all ports in interactive mode.
 343  */
 344 uint8_t no_device_start = 0;
 345
 346 /*
 347  * Enable link status change notification
 348  */
 349 uint8_t lsc_interrupt = 1; /* enabled by default */
 350
 351 /*
 352  * Enable device removal notification.
 353  */
 354 uint8_t rmv_interrupt = 1; /* enabled by default */
 355
 356 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 357
 358 /* After attach, port setup is called on event or by iterator */
 359 bool setup_on_probe_event = true;
 360
 361 /* Pretty printing of ethdev events */
 362 static const char * const eth_event_desc[] = {
 363         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 364         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 365         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 366         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 367         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 368         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 369         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 370         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 371         [RTE_ETH_EVENT_NEW] = "device probed",
 372         [RTE_ETH_EVENT_DESTROY] = "device released",
 373         [RTE_ETH_EVENT_MAX] = NULL,
 374 };
 375
 376 /*
 377  * Display or mask ether events
 378  * Default to all events except VF_MBOX
 379  */
 380 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 381                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 382                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 383                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 384                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 385                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 386                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 387 /*
 388  * Decide if all memory are locked for performance.
 389  */
 390 int do_mlockall = 0;
 391
 392 /*
 393  * NIC bypass mode configuration options.
 394  */
 395
 396 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 397 /* The NIC bypass watchdog timeout. */
 398 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 399 #endif
 400
 401
 402 #ifdef RTE_LIBRTE_LATENCY_STATS
 403
 404 /*
 405  * Set when latency stats is enabled in the commandline
 406  */
 407 uint8_t latencystats_enabled;
 408
 409 /*
 410  * Lcore ID to serive latency statistics.
 411  */
 412 lcoreid_t latencystats_lcore_id = -1;
 413
 414 #endif
 415
 416 /*
 417  * Ethernet device configuration.
 418  */
 419 struct rte_eth_rxmode rx_mode = {
 420         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 421                 /**< Default maximum frame length. */
 422 };
 423
 424 struct rte_eth_txmode tx_mode = {
 425         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 426 };
 427
 428 struct rte_fdir_conf fdir_conf = {
 429         .mode = RTE_FDIR_MODE_NONE,
 430         .pballoc = RTE_FDIR_PBALLOC_64K,
 431         .status = RTE_FDIR_REPORT_STATUS,
 432         .mask = {
 433                 .vlan_tci_mask = 0xFFEF,
 434                 .ipv4_mask     = {
 435                         .src_ip = 0xFFFFFFFF,
 436                         .dst_ip = 0xFFFFFFFF,
 437                 },
 438                 .ipv6_mask     = {
 439                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 440                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 441                 },
 442                 .src_port_mask = 0xFFFF,
 443                 .dst_port_mask = 0xFFFF,
 444                 .mac_addr_byte_mask = 0xFF,
 445                 .tunnel_type_mask = 1,
 446                 .tunnel_id_mask = 0xFFFFFFFF,
 447         },
 448         .drop_queue = 127,
 449 };
 450
 451 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 452
 453 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 454 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 455
 456 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 457 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 458
 459 uint16_t nb_tx_queue_stats_mappings = 0;
 460 uint16_t nb_rx_queue_stats_mappings = 0;
 461
 462 /*
 463  * Display zero values by default for xstats
 464  */
 465 uint8_t xstats_hide_zero;
 466
 467 unsigned int num_sockets = 0;
 468 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 469
 470 #ifdef RTE_LIBRTE_BITRATE
 471 /* Bitrate statistics */
 472 struct rte_stats_bitrates *bitrate_data;
 473 lcoreid_t bitrate_lcore_id;
 474 uint8_t bitrate_enabled;
 475 #endif
 476
 477 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 478 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 479
 480 /* Forward function declarations */
 481 static void setup_attached_port(portid_t pi);
 482 static void map_port_queue_stats_mapping_registers(portid_t pi,
 483                                                    struct rte_port *port);
 484 static void check_all_ports_link_status(uint32_t port_mask);
 485 static int eth_event_callback(portid_t port_id,
 486                               enum rte_eth_event_type type,
 487                               void *param, void *ret_param);
 488 static void dev_event_callback(const char *device_name,
 489                                 enum rte_dev_event_type type,
 490                                 void *param);
 491
 492 /*
 493  * Check if all the ports are started.
 494  * If yes, return positive value. If not, return zero.
 495  */
 496 static int all_ports_started(void);
 497
 498 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 499 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 500
 501 /*
 502  * Helper function to check if socket is already discovered.
 503  * If yes, return positive value. If not, return zero.
 504  */
 505 int
 506 new_socket_id(unsigned int socket_id)
 507 {
 508         unsigned int i;
 509
 510         for (i = 0; i < num_sockets; i++) {
 511                 if (socket_ids[i] == socket_id)
 512                         return 0;
 513         }
 514         return 1;
 515 }
 516
 517 /*
 518  * Setup default configuration.
 519  */
 520 static void
 521 set_default_fwd_lcores_config(void)
 522 {
 523         unsigned int i;
 524         unsigned int nb_lc;
 525         unsigned int sock_num;
 526
 527         nb_lc = 0;
 528         for (i = 0; i < RTE_MAX_LCORE; i++) {
 529                 if (!rte_lcore_is_enabled(i))
 530                         continue;
 531                 sock_num = rte_lcore_to_socket_id(i);
 532                 if (new_socket_id(sock_num)) {
 533                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 534                                 rte_exit(EXIT_FAILURE,
 535                                          "Total sockets greater than %u\n",
 536                                          RTE_MAX_NUMA_NODES);
 537                         }
 538                         socket_ids[num_sockets++] = sock_num;
 539                 }
 540                 if (i == rte_get_master_lcore())
 541                         continue;
 542                 fwd_lcores_cpuids[nb_lc++] = i;
 543         }
 544         nb_lcores = (lcoreid_t) nb_lc;
 545         nb_cfg_lcores = nb_lcores;
 546         nb_fwd_lcores = 1;
 547 }
 548
 549 static void
 550 set_def_peer_eth_addrs(void)
 551 {
 552         portid_t i;
 553
 554         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 555                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 556                 peer_eth_addrs[i].addr_bytes[5] = i;
 557         }
 558 }
 559
 560 static void
 561 set_default_fwd_ports_config(void)
 562 {
 563         portid_t pt_id;
 564         int i = 0;
 565
 566         RTE_ETH_FOREACH_DEV(pt_id) {
 567                 fwd_ports_ids[i++] = pt_id;
 568
 569                 /* Update sockets info according to the attached device */
 570                 int socket_id = rte_eth_dev_socket_id(pt_id);
 571                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 572                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 573                                 rte_exit(EXIT_FAILURE,
 574                                          "Total sockets greater than %u\n",
 575                                          RTE_MAX_NUMA_NODES);
 576                         }
 577                         socket_ids[num_sockets++] = socket_id;
 578                 }
 579         }
 580
 581         nb_cfg_ports = nb_ports;
 582         nb_fwd_ports = nb_ports;
 583 }
 584
 585 void
 586 set_def_fwd_config(void)
 587 {
 588         set_default_fwd_lcores_config();
 589         set_def_peer_eth_addrs();
 590         set_default_fwd_ports_config();
 591 }
 592
 593 /* extremely pessimistic estimation of memory required to create a mempool */
 594 static int
 595 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 596 {
 597         unsigned int n_pages, mbuf_per_pg, leftover;
 598         uint64_t total_mem, mbuf_mem, obj_sz;
 599
 600         /* there is no good way to predict how much space the mempool will
 601          * occupy because it will allocate chunks on the fly, and some of those
 602          * will come from default DPDK memory while some will come from our
 603          * external memory, so just assume 128MB will be enough for everyone.
 604          */
 605         uint64_t hdr_mem = 128 << 20;
 606
 607         /* account for possible non-contiguousness */
 608         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 609         if (obj_sz > pgsz) {
 610                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 611                 return -1;
 612         }
 613
 614         mbuf_per_pg = pgsz / obj_sz;
 615         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 616         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 617
 618         mbuf_mem = n_pages * pgsz;
 619
 620         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 621
 622         if (total_mem > SIZE_MAX) {
 623                 TESTPMD_LOG(ERR, "Memory size too big\n");
 624                 return -1;
 625         }
 626         *out = (size_t)total_mem;
 627
 628         return 0;
 629 }
 630
 631 static int
 632 pagesz_flags(uint64_t page_sz)
 633 {
 634         /* as per mmap() manpage, all page sizes are log2 of page size
 635          * shifted by MAP_HUGE_SHIFT
 636          */
 637         int log2 = rte_log2_u64(page_sz);
 638
 639         return (log2 << HUGE_SHIFT);
 640 }
 641
 642 static void *
 643 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 644 {
 645         void *addr;
 646         int flags;
 647
 648         /* allocate anonymous hugepages */
 649         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 650         if (huge)
 651                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 652
 653         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 654         if (addr == MAP_FAILED)
 655                 return NULL;
 656
 657         return addr;
 658 }
 659
 660 struct extmem_param {
 661         void *addr;
 662         size_t len;
 663         size_t pgsz;
 664         rte_iova_t *iova_table;
 665         unsigned int iova_table_len;
 666 };
 667
 668 static int
 669 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 670                 bool huge)
 671 {
 672         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 673                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 674         unsigned int cur_page, n_pages, pgsz_idx;
 675         size_t mem_sz, cur_pgsz;
 676         rte_iova_t *iovas = NULL;
 677         void *addr;
 678         int ret;
 679
 680         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 681                 /* skip anything that is too big */
 682                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 683                         continue;
 684
 685                 cur_pgsz = pgsizes[pgsz_idx];
 686
 687                 /* if we were told not to allocate hugepages, override */
 688                 if (!huge)
 689                         cur_pgsz = sysconf(_SC_PAGESIZE);
 690
 691                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 692                 if (ret < 0) {
 693                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 694                         return -1;
 695                 }
 696
 697                 /* allocate our memory */
 698                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 699
 700                 /* if we couldn't allocate memory with a specified page size,
 701                  * that doesn't mean we can't do it with other page sizes, so
 702                  * try another one.
 703                  */
 704                 if (addr == NULL)
 705                         continue;
 706
 707                 /* store IOVA addresses for every page in this memory area */
 708                 n_pages = mem_sz / cur_pgsz;
 709
 710                 iovas = malloc(sizeof(*iovas) * n_pages);
 711
 712                 if (iovas == NULL) {
 713                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 714                         goto fail;
 715                 }
 716                 /* lock memory if it's not huge pages */
 717                 if (!huge)
 718                         mlock(addr, mem_sz);
 719
 720                 /* populate IOVA addresses */
 721                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 722                         rte_iova_t iova;
 723                         size_t offset;
 724                         void *cur;
 725
 726                         offset = cur_pgsz * cur_page;
 727                         cur = RTE_PTR_ADD(addr, offset);
 728
 729                         /* touch the page before getting its IOVA */
 730                         *(volatile char *)cur = 0;
 731
 732                         iova = rte_mem_virt2iova(cur);
 733
 734                         iovas[cur_page] = iova;
 735                 }
 736
 737                 break;
 738         }
 739         /* if we couldn't allocate anything */
 740         if (iovas == NULL)
 741                 return -1;
 742
 743         param->addr = addr;
 744         param->len = mem_sz;
 745         param->pgsz = cur_pgsz;
 746         param->iova_table = iovas;
 747         param->iova_table_len = n_pages;
 748
 749         return 0;
 750 fail:
 751         if (iovas)
 752                 free(iovas);
 753         if (addr)
 754                 munmap(addr, mem_sz);
 755
 756         return -1;
 757 }
 758
 759 static int
 760 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 761 {
 762         struct extmem_param param;
 763         int socket_id, ret;
 764
 765         memset(&param, 0, sizeof(param));
 766
 767         /* check if our heap exists */
 768         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 769         if (socket_id < 0) {
 770                 /* create our heap */
 771                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 772                 if (ret < 0) {
 773                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 774                         return -1;
 775                 }
 776         }
 777
 778         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 779         if (ret < 0) {
 780                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 781                 return -1;
 782         }
 783
 784         /* we now have a valid memory area, so add it to heap */
 785         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 786                         param.addr, param.len, param.iova_table,
 787                         param.iova_table_len, param.pgsz);
 788
 789         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 790
 791         /* not needed any more */
 792         free(param.iova_table);
 793
 794         if (ret < 0) {
 795                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 796                 munmap(param.addr, param.len);
 797                 return -1;
 798         }
 799
 800         /* success */
 801
 802         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 803                         param.len >> 20);
 804
 805         return 0;
 806 }
 807 static void
 808 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 809              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 810 {
 811         uint16_t pid = 0;
 812         int ret;
 813
 814         RTE_ETH_FOREACH_DEV(pid) {
 815                 struct rte_eth_dev *dev =
 816                         &rte_eth_devices[pid];
 817
 818                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 819                                         memhdr->len);
 820                 if (ret) {
 821                         TESTPMD_LOG(DEBUG,
 822                                     "unable to DMA unmap addr 0x%p "
 823                                     "for device %s\n",
 824                                     memhdr->addr, dev->data->name);
 825                 }
 826         }
 827         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 828         if (ret) {
 829                 TESTPMD_LOG(DEBUG,
 830                             "unable to un-register addr 0x%p\n", memhdr->addr);
 831         }
 832 }
 833
 834 static void
 835 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 836            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 837 {
 838         uint16_t pid = 0;
 839         size_t page_size = sysconf(_SC_PAGESIZE);
 840         int ret;
 841
 842         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 843                                   page_size);
 844         if (ret) {
 845                 TESTPMD_LOG(DEBUG,
 846                             "unable to register addr 0x%p\n", memhdr->addr);
 847                 return;
 848         }
 849         RTE_ETH_FOREACH_DEV(pid) {
 850                 struct rte_eth_dev *dev =
 851                         &rte_eth_devices[pid];
 852
 853                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 854                                       memhdr->len);
 855                 if (ret) {
 856                         TESTPMD_LOG(DEBUG,
 857                                     "unable to DMA map addr 0x%p "
 858                                     "for device %s\n",
 859                                     memhdr->addr, dev->data->name);
 860                 }
 861         }
 862 }
 863
 864 /*
 865  * Configuration initialisation done once at init time.
 866  */
 867 static struct rte_mempool *
 868 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 869                  unsigned int socket_id)
 870 {
 871         char pool_name[RTE_MEMPOOL_NAMESIZE];
 872         struct rte_mempool *rte_mp = NULL;
 873         uint32_t mb_size;
 874
 875         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 876         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 877
 878         TESTPMD_LOG(INFO,
 879                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 880                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 881
 882         switch (mp_alloc_type) {
 883         case MP_ALLOC_NATIVE:
 884                 {
 885                         /* wrapper to rte_mempool_create() */
 886                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 887                                         rte_mbuf_best_mempool_ops());
 888                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 889                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 890                         break;
 891                 }
 892         case MP_ALLOC_ANON:
 893                 {
 894                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 895                                 mb_size, (unsigned int) mb_mempool_cache,
 896                                 sizeof(struct rte_pktmbuf_pool_private),
 897                                 socket_id, mempool_flags);
 898                         if (rte_mp == NULL)
 899                                 goto err;
 900
 901                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 902                                 rte_mempool_free(rte_mp);
 903                                 rte_mp = NULL;
 904                                 goto err;
 905                         }
 906                         rte_pktmbuf_pool_init(rte_mp, NULL);
 907                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 908                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 909                         break;
 910                 }
 911         case MP_ALLOC_XMEM:
 912         case MP_ALLOC_XMEM_HUGE:
 913                 {
 914                         int heap_socket;
 915                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 916
 917                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 918                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 919
 920                         heap_socket =
 921                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 922                         if (heap_socket < 0)
 923                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 924
 925                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 926                                         rte_mbuf_best_mempool_ops());
 927                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 928                                         mb_mempool_cache, 0, mbuf_seg_size,
 929                                         heap_socket);
 930                         break;
 931                 }
 932         default:
 933                 {
 934                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 935                 }
 936         }
 937
 938 err:
 939         if (rte_mp == NULL) {
 940                 rte_exit(EXIT_FAILURE,
 941                         "Creation of mbuf pool for socket %u failed: %s\n",
 942                         socket_id, rte_strerror(rte_errno));
 943         } else if (verbose_level > 0) {
 944                 rte_mempool_dump(stdout, rte_mp);
 945         }
 946         return rte_mp;
 947 }
 948
 949 /*
 950  * Check given socket id is valid or not with NUMA mode,
 951  * if valid, return 0, else return -1
 952  */
 953 static int
 954 check_socket_id(const unsigned int socket_id)
 955 {
 956         static int warning_once = 0;
 957
 958         if (new_socket_id(socket_id)) {
 959                 if (!warning_once && numa_support)
 960                         printf("Warning: NUMA should be configured manually by"
 961                                " using --port-numa-config and"
 962                                " --ring-numa-config parameters along with"
 963                                " --numa.\n");
 964                 warning_once = 1;
 965                 return -1;
 966         }
 967         return 0;
 968 }
 969
 970 /*
 971  * Get the allowed maximum number of RX queues.
 972  * *pid return the port id which has minimal value of
 973  * max_rx_queues in all ports.
 974  */
 975 queueid_t
 976 get_allowed_max_nb_rxq(portid_t *pid)
 977 {
 978         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
 979         bool max_rxq_valid = false;
 980         portid_t pi;
 981         struct rte_eth_dev_info dev_info;
 982
 983         RTE_ETH_FOREACH_DEV(pi) {
 984                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
 985                         continue;
 986
 987                 max_rxq_valid = true;
 988                 if (dev_info.max_rx_queues < allowed_max_rxq) {
 989                         allowed_max_rxq = dev_info.max_rx_queues;
 990                         *pid = pi;
 991                 }
 992         }
 993         return max_rxq_valid ? allowed_max_rxq : 0;
 994 }
 995
 996 /*
 997  * Check input rxq is valid or not.
 998  * If input rxq is not greater than any of maximum number
 999  * of RX queues of all ports, it is valid.
1000  * if valid, return 0, else return -1
1001  */
1002 int
1003 check_nb_rxq(queueid_t rxq)
1004 {
1005         queueid_t allowed_max_rxq;
1006         portid_t pid = 0;
1007
1008         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1009         if (rxq > allowed_max_rxq) {
1010                 printf("Fail: input rxq (%u) can't be greater "
1011                        "than max_rx_queues (%u) of port %u\n",
1012                        rxq,
1013                        allowed_max_rxq,
1014                        pid);
1015                 return -1;
1016         }
1017         return 0;
1018 }
1019
1020 /*
1021  * Get the allowed maximum number of TX queues.
1022  * *pid return the port id which has minimal value of
1023  * max_tx_queues in all ports.
1024  */
1025 queueid_t
1026 get_allowed_max_nb_txq(portid_t *pid)
1027 {
1028         queueid_t allowed_max_txq = MAX_QUEUE_ID;
1029         bool max_txq_valid = false;
1030         portid_t pi;
1031         struct rte_eth_dev_info dev_info;
1032
1033         RTE_ETH_FOREACH_DEV(pi) {
1034                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1035                         continue;
1036
1037                 max_txq_valid = true;
1038                 if (dev_info.max_tx_queues < allowed_max_txq) {
1039                         allowed_max_txq = dev_info.max_tx_queues;
1040                         *pid = pi;
1041                 }
1042         }
1043         return max_txq_valid ? allowed_max_txq : 0;
1044 }
1045
1046 /*
1047  * Check input txq is valid or not.
1048  * If input txq is not greater than any of maximum number
1049  * of TX queues of all ports, it is valid.
1050  * if valid, return 0, else return -1
1051  */
1052 int
1053 check_nb_txq(queueid_t txq)
1054 {
1055         queueid_t allowed_max_txq;
1056         portid_t pid = 0;
1057
1058         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1059         if (txq > allowed_max_txq) {
1060                 printf("Fail: input txq (%u) can't be greater "
1061                        "than max_tx_queues (%u) of port %u\n",
1062                        txq,
1063                        allowed_max_txq,
1064                        pid);
1065                 return -1;
1066         }
1067         return 0;
1068 }
1069
1070 static void
1071 init_config(void)
1072 {
1073         portid_t pid;
1074         struct rte_port *port;
1075         struct rte_mempool *mbp;
1076         unsigned int nb_mbuf_per_pool;
1077         lcoreid_t  lc_id;
1078         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1079         struct rte_gro_param gro_param;
1080         uint32_t gso_types;
1081         uint16_t data_size;
1082         bool warning = 0;
1083         int k;
1084         int ret;
1085
1086         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1087
1088         /* Configuration of logical cores. */
1089         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1090                                 sizeof(struct fwd_lcore *) * nb_lcores,
1091                                 RTE_CACHE_LINE_SIZE);
1092         if (fwd_lcores == NULL) {
1093                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1094                                                         "failed\n", nb_lcores);
1095         }
1096         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1097                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1098                                                sizeof(struct fwd_lcore),
1099                                                RTE_CACHE_LINE_SIZE);
1100                 if (fwd_lcores[lc_id] == NULL) {
1101                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1102                                                                 "failed\n");
1103                 }
1104                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1105         }
1106
1107         RTE_ETH_FOREACH_DEV(pid) {
1108                 port = &ports[pid];
1109                 /* Apply default TxRx configuration for all ports */
1110                 port->dev_conf.txmode = tx_mode;
1111                 port->dev_conf.rxmode = rx_mode;
1112
1113                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1114                 if (ret != 0)
1115                         rte_exit(EXIT_FAILURE,
1116                                  "rte_eth_dev_info_get() failed\n");
1117
1118                 if (!(port->dev_info.tx_offload_capa &
1119                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1120                         port->dev_conf.txmode.offloads &=
1121                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1122                 if (numa_support) {
1123                         if (port_numa[pid] != NUMA_NO_CONFIG)
1124                                 port_per_socket[port_numa[pid]]++;
1125                         else {
1126                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1127
1128                                 /*
1129                                  * if socket_id is invalid,
1130                                  * set to the first available socket.
1131                                  */
1132                                 if (check_socket_id(socket_id) < 0)
1133                                         socket_id = socket_ids[0];
1134                                 port_per_socket[socket_id]++;
1135                         }
1136                 }
1137
1138                 /* Apply Rx offloads configuration */
1139                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1140                         port->rx_conf[k].offloads =
1141                                 port->dev_conf.rxmode.offloads;
1142                 /* Apply Tx offloads configuration */
1143                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1144                         port->tx_conf[k].offloads =
1145                                 port->dev_conf.txmode.offloads;
1146
1147                 /* set flag to initialize port/queue */
1148                 port->need_reconfig = 1;
1149                 port->need_reconfig_queues = 1;
1150                 port->tx_metadata = 0;
1151
1152                 /* Check for maximum number of segments per MTU. Accordingly
1153                  * update the mbuf data size.
1154                  */
1155                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1156                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1157                         data_size = rx_mode.max_rx_pkt_len /
1158                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1159
1160                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1161                                                         mbuf_data_size) {
1162                                 mbuf_data_size = data_size +
1163                                                  RTE_PKTMBUF_HEADROOM;
1164                                 warning = 1;
1165                         }
1166                 }
1167         }
1168
1169         if (warning)
1170                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1171                             mbuf_data_size);
1172
1173         /*
1174          * Create pools of mbuf.
1175          * If NUMA support is disabled, create a single pool of mbuf in
1176          * socket 0 memory by default.
1177          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1178          *
1179          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1180          * nb_txd can be configured at run time.
1181          */
1182         if (param_total_num_mbufs)
1183                 nb_mbuf_per_pool = param_total_num_mbufs;
1184         else {
1185                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1186                         (nb_lcores * mb_mempool_cache) +
1187                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1188                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1189         }
1190
1191         if (numa_support) {
1192                 uint8_t i;
1193
1194                 for (i = 0; i < num_sockets; i++)
1195                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1196                                                        nb_mbuf_per_pool,
1197                                                        socket_ids[i]);
1198         } else {
1199                 if (socket_num == UMA_NO_CONFIG)
1200                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1201                                                        nb_mbuf_per_pool, 0);
1202                 else
1203                         mempools[socket_num] = mbuf_pool_create
1204                                                         (mbuf_data_size,
1205                                                          nb_mbuf_per_pool,
1206                                                          socket_num);
1207         }
1208
1209         init_port_config();
1210
1211         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1212                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1213         /*
1214          * Records which Mbuf pool to use by each logical core, if needed.
1215          */
1216         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1217                 mbp = mbuf_pool_find(
1218                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1219
1220                 if (mbp == NULL)
1221                         mbp = mbuf_pool_find(0);
1222                 fwd_lcores[lc_id]->mbp = mbp;
1223                 /* initialize GSO context */
1224                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1225                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1226                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1227                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1228                         RTE_ETHER_CRC_LEN;
1229                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1230         }
1231
1232         /* Configuration of packet forwarding streams. */
1233         if (init_fwd_streams() < 0)
1234                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1235
1236         fwd_config_setup();
1237
1238         /* create a gro context for each lcore */
1239         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1240         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1241         gro_param.max_item_per_flow = MAX_PKT_BURST;
1242         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1243                 gro_param.socket_id = rte_lcore_to_socket_id(
1244                                 fwd_lcores_cpuids[lc_id]);
1245                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1246                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1247                         rte_exit(EXIT_FAILURE,
1248                                         "rte_gro_ctx_create() failed\n");
1249                 }
1250         }
1251
1252 #if defined RTE_LIBRTE_PMD_SOFTNIC
1253         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1254                 RTE_ETH_FOREACH_DEV(pid) {
1255                         port = &ports[pid];
1256                         const char *driver = port->dev_info.driver_name;
1257
1258                         if (strcmp(driver, "net_softnic") == 0)
1259                                 port->softport.fwd_lcore_arg = fwd_lcores;
1260                 }
1261         }
1262 #endif
1263
1264 }
1265
1266
1267 void
1268 reconfig(portid_t new_port_id, unsigned socket_id)
1269 {
1270         struct rte_port *port;
1271         int ret;
1272
1273         /* Reconfiguration of Ethernet ports. */
1274         port = &ports[new_port_id];
1275
1276         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1277         if (ret != 0)
1278                 return;
1279
1280         /* set flag to initialize port/queue */
1281         port->need_reconfig = 1;
1282         port->need_reconfig_queues = 1;
1283         port->socket_id = socket_id;
1284
1285         init_port_config();
1286 }
1287
1288
1289 int
1290 init_fwd_streams(void)
1291 {
1292         portid_t pid;
1293         struct rte_port *port;
1294         streamid_t sm_id, nb_fwd_streams_new;
1295         queueid_t q;
1296
1297         /* set socket id according to numa or not */
1298         RTE_ETH_FOREACH_DEV(pid) {
1299                 port = &ports[pid];
1300                 if (nb_rxq > port->dev_info.max_rx_queues) {
1301                         printf("Fail: nb_rxq(%d) is greater than "
1302                                 "max_rx_queues(%d)\n", nb_rxq,
1303                                 port->dev_info.max_rx_queues);
1304                         return -1;
1305                 }
1306                 if (nb_txq > port->dev_info.max_tx_queues) {
1307                         printf("Fail: nb_txq(%d) is greater than "
1308                                 "max_tx_queues(%d)\n", nb_txq,
1309                                 port->dev_info.max_tx_queues);
1310                         return -1;
1311                 }
1312                 if (numa_support) {
1313                         if (port_numa[pid] != NUMA_NO_CONFIG)
1314                                 port->socket_id = port_numa[pid];
1315                         else {
1316                                 port->socket_id = rte_eth_dev_socket_id(pid);
1317
1318                                 /*
1319                                  * if socket_id is invalid,
1320                                  * set to the first available socket.
1321                                  */
1322                                 if (check_socket_id(port->socket_id) < 0)
1323                                         port->socket_id = socket_ids[0];
1324                         }
1325                 }
1326                 else {
1327                         if (socket_num == UMA_NO_CONFIG)
1328                                 port->socket_id = 0;
1329                         else
1330                                 port->socket_id = socket_num;
1331                 }
1332         }
1333
1334         q = RTE_MAX(nb_rxq, nb_txq);
1335         if (q == 0) {
1336                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1337                 return -1;
1338         }
1339         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1340         if (nb_fwd_streams_new == nb_fwd_streams)
1341                 return 0;
1342         /* clear the old */
1343         if (fwd_streams != NULL) {
1344                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1345                         if (fwd_streams[sm_id] == NULL)
1346                                 continue;
1347                         rte_free(fwd_streams[sm_id]);
1348                         fwd_streams[sm_id] = NULL;
1349                 }
1350                 rte_free(fwd_streams);
1351                 fwd_streams = NULL;
1352         }
1353
1354         /* init new */
1355         nb_fwd_streams = nb_fwd_streams_new;
1356         if (nb_fwd_streams) {
1357                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1358                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1359                         RTE_CACHE_LINE_SIZE);
1360                 if (fwd_streams == NULL)
1361                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1362                                  " (struct fwd_stream *)) failed\n",
1363                                  nb_fwd_streams);
1364
1365                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1366                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1367                                 " struct fwd_stream", sizeof(struct fwd_stream),
1368                                 RTE_CACHE_LINE_SIZE);
1369                         if (fwd_streams[sm_id] == NULL)
1370                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1371                                          "(struct fwd_stream) failed\n");
1372                 }
1373         }
1374
1375         return 0;
1376 }
1377
1378 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1379 static void
1380 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1381 {
1382         unsigned int total_burst;
1383         unsigned int nb_burst;
1384         unsigned int burst_stats[3];
1385         uint16_t pktnb_stats[3];
1386         uint16_t nb_pkt;
1387         int burst_percent[3];
1388
1389         /*
1390          * First compute the total number of packet bursts and the
1391          * two highest numbers of bursts of the same number of packets.
1392          */
1393         total_burst = 0;
1394         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1395         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1396         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1397                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1398                 if (nb_burst == 0)
1399                         continue;
1400                 total_burst += nb_burst;
1401                 if (nb_burst > burst_stats[0]) {
1402                         burst_stats[1] = burst_stats[0];
1403                         pktnb_stats[1] = pktnb_stats[0];
1404                         burst_stats[0] = nb_burst;
1405                         pktnb_stats[0] = nb_pkt;
1406                 } else if (nb_burst > burst_stats[1]) {
1407                         burst_stats[1] = nb_burst;
1408                         pktnb_stats[1] = nb_pkt;
1409                 }
1410         }
1411         if (total_burst == 0)
1412                 return;
1413         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1414         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1415                burst_percent[0], (int) pktnb_stats[0]);
1416         if (burst_stats[0] == total_burst) {
1417                 printf("]\n");
1418                 return;
1419         }
1420         if (burst_stats[0] + burst_stats[1] == total_burst) {
1421                 printf(" + %d%% of %d pkts]\n",
1422                        100 - burst_percent[0], pktnb_stats[1]);
1423                 return;
1424         }
1425         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1426         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1427         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1428                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1429                 return;
1430         }
1431         printf(" + %d%% of %d pkts + %d%% of others]\n",
1432                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1433 }
1434 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1435
1436 static void
1437 fwd_stream_stats_display(streamid_t stream_id)
1438 {
1439         struct fwd_stream *fs;
1440         static const char *fwd_top_stats_border = "-------";
1441
1442         fs = fwd_streams[stream_id];
1443         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1444             (fs->fwd_dropped == 0))
1445                 return;
1446         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1447                "TX Port=%2d/Queue=%2d %s\n",
1448                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1449                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1450         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1451                " TX-dropped: %-14"PRIu64,
1452                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1453
1454         /* if checksum mode */
1455         if (cur_fwd_eng == &csum_fwd_engine) {
1456                 printf("  RX- bad IP checksum: %-14"PRIu64
1457                        "  Rx- bad L4 checksum: %-14"PRIu64
1458                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1459                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1460                         fs->rx_bad_outer_l4_csum);
1461         } else {
1462                 printf("\n");
1463         }
1464
1465 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1466         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1467         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1468 #endif
1469 }
1470
1471 void
1472 fwd_stats_display(void)
1473 {
1474         static const char *fwd_stats_border = "----------------------";
1475         static const char *acc_stats_border = "+++++++++++++++";
1476         struct {
1477                 struct fwd_stream *rx_stream;
1478                 struct fwd_stream *tx_stream;
1479                 uint64_t tx_dropped;
1480                 uint64_t rx_bad_ip_csum;
1481                 uint64_t rx_bad_l4_csum;
1482                 uint64_t rx_bad_outer_l4_csum;
1483         } ports_stats[RTE_MAX_ETHPORTS];
1484         uint64_t total_rx_dropped = 0;
1485         uint64_t total_tx_dropped = 0;
1486         uint64_t total_rx_nombuf = 0;
1487         struct rte_eth_stats stats;
1488 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1489         uint64_t fwd_cycles = 0;
1490 #endif
1491         uint64_t total_recv = 0;
1492         uint64_t total_xmit = 0;
1493         struct rte_port *port;
1494         streamid_t sm_id;
1495         portid_t pt_id;
1496         int i;
1497
1498         memset(ports_stats, 0, sizeof(ports_stats));
1499
1500         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1501                 struct fwd_stream *fs = fwd_streams[sm_id];
1502
1503                 if (cur_fwd_config.nb_fwd_streams >
1504                     cur_fwd_config.nb_fwd_ports) {
1505                         fwd_stream_stats_display(sm_id);
1506                 } else {
1507                         ports_stats[fs->tx_port].tx_stream = fs;
1508                         ports_stats[fs->rx_port].rx_stream = fs;
1509                 }
1510
1511                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1512
1513                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1514                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1515                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1516                                 fs->rx_bad_outer_l4_csum;
1517
1518 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1519                 fwd_cycles += fs->core_cycles;
1520 #endif
1521         }
1522         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1523                 uint8_t j;
1524
1525                 pt_id = fwd_ports_ids[i];
1526                 port = &ports[pt_id];
1527
1528                 rte_eth_stats_get(pt_id, &stats);
1529                 stats.ipackets -= port->stats.ipackets;
1530                 stats.opackets -= port->stats.opackets;
1531                 stats.ibytes -= port->stats.ibytes;
1532                 stats.obytes -= port->stats.obytes;
1533                 stats.imissed -= port->stats.imissed;
1534                 stats.oerrors -= port->stats.oerrors;
1535                 stats.rx_nombuf -= port->stats.rx_nombuf;
1536
1537                 total_recv += stats.ipackets;
1538                 total_xmit += stats.opackets;
1539                 total_rx_dropped += stats.imissed;
1540                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1541                 total_tx_dropped += stats.oerrors;
1542                 total_rx_nombuf  += stats.rx_nombuf;
1543
1544                 printf("\n  %s Forward statistics for port %-2d %s\n",
1545                        fwd_stats_border, pt_id, fwd_stats_border);
1546
1547                 if (!port->rx_queue_stats_mapping_enabled &&
1548                     !port->tx_queue_stats_mapping_enabled) {
1549                         printf("  RX-packets: %-14"PRIu64
1550                                " RX-dropped: %-14"PRIu64
1551                                "RX-total: %-"PRIu64"\n",
1552                                stats.ipackets, stats.imissed,
1553                                stats.ipackets + stats.imissed);
1554
1555                         if (cur_fwd_eng == &csum_fwd_engine)
1556                                 printf("  Bad-ipcsum: %-14"PRIu64
1557                                        " Bad-l4csum: %-14"PRIu64
1558                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1559                                        ports_stats[pt_id].rx_bad_ip_csum,
1560                                        ports_stats[pt_id].rx_bad_l4_csum,
1561                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1562                         if (stats.ierrors + stats.rx_nombuf > 0) {
1563                                 printf("  RX-error: %-"PRIu64"\n",
1564                                        stats.ierrors);
1565                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1566                                        stats.rx_nombuf);
1567                         }
1568
1569                         printf("  TX-packets: %-14"PRIu64
1570                                " TX-dropped: %-14"PRIu64
1571                                "TX-total: %-"PRIu64"\n",
1572                                stats.opackets, ports_stats[pt_id].tx_dropped,
1573                                stats.opackets + ports_stats[pt_id].tx_dropped);
1574                 } else {
1575                         printf("  RX-packets:             %14"PRIu64
1576                                "    RX-dropped:%14"PRIu64
1577                                "    RX-total:%14"PRIu64"\n",
1578                                stats.ipackets, stats.imissed,
1579                                stats.ipackets + stats.imissed);
1580
1581                         if (cur_fwd_eng == &csum_fwd_engine)
1582                                 printf("  Bad-ipcsum:%14"PRIu64
1583                                        "    Bad-l4csum:%14"PRIu64
1584                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1585                                        ports_stats[pt_id].rx_bad_ip_csum,
1586                                        ports_stats[pt_id].rx_bad_l4_csum,
1587                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1588                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1589                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1590                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1591                                        stats.rx_nombuf);
1592                         }
1593
1594                         printf("  TX-packets:             %14"PRIu64
1595                                "    TX-dropped:%14"PRIu64
1596                                "    TX-total:%14"PRIu64"\n",
1597                                stats.opackets, ports_stats[pt_id].tx_dropped,
1598                                stats.opackets + ports_stats[pt_id].tx_dropped);
1599                 }
1600
1601 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1602                 if (ports_stats[pt_id].rx_stream)
1603                         pkt_burst_stats_display("RX",
1604                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1605                 if (ports_stats[pt_id].tx_stream)
1606                         pkt_burst_stats_display("TX",
1607                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1608 #endif
1609
1610                 if (port->rx_queue_stats_mapping_enabled) {
1611                         printf("\n");
1612                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1613                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1614                                        "     RX-errors:%14"PRIu64
1615                                        "    RX-bytes:%14"PRIu64"\n",
1616                                        j, stats.q_ipackets[j],
1617                                        stats.q_errors[j], stats.q_ibytes[j]);
1618                         }
1619                         printf("\n");
1620                 }
1621                 if (port->tx_queue_stats_mapping_enabled) {
1622                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1623                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1624                                        "                                 TX-bytes:%14"
1625                                        PRIu64"\n",
1626                                        j, stats.q_opackets[j],
1627                                        stats.q_obytes[j]);
1628                         }
1629                 }
1630
1631                 printf("  %s--------------------------------%s\n",
1632                        fwd_stats_border, fwd_stats_border);
1633         }
1634
1635         printf("\n  %s Accumulated forward statistics for all ports"
1636                "%s\n",
1637                acc_stats_border, acc_stats_border);
1638         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1639                "%-"PRIu64"\n"
1640                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1641                "%-"PRIu64"\n",
1642                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1643                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1644         if (total_rx_nombuf > 0)
1645                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1646         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1647                "%s\n",
1648                acc_stats_border, acc_stats_border);
1649 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1650         if (total_recv > 0)
1651                 printf("\n  CPU cycles/packet=%u (total cycles="
1652                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1653                        (unsigned int)(fwd_cycles / total_recv),
1654                        fwd_cycles, total_recv);
1655 #endif
1656 }
1657
1658 void
1659 fwd_stats_reset(void)
1660 {
1661         streamid_t sm_id;
1662         portid_t pt_id;
1663         int i;
1664
1665         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1666                 pt_id = fwd_ports_ids[i];
1667                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1668         }
1669         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1670                 struct fwd_stream *fs = fwd_streams[sm_id];
1671
1672                 fs->rx_packets = 0;
1673                 fs->tx_packets = 0;
1674                 fs->fwd_dropped = 0;
1675                 fs->rx_bad_ip_csum = 0;
1676                 fs->rx_bad_l4_csum = 0;
1677                 fs->rx_bad_outer_l4_csum = 0;
1678
1679 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1680                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1681                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1682 #endif
1683 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1684                 fs->core_cycles = 0;
1685 #endif
1686         }
1687 }
1688
1689 static void
1690 flush_fwd_rx_queues(void)
1691 {
1692         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1693         portid_t  rxp;
1694         portid_t port_id;
1695         queueid_t rxq;
1696         uint16_t  nb_rx;
1697         uint16_t  i;
1698         uint8_t   j;
1699         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1700         uint64_t timer_period;
1701
1702         /* convert to number of cycles */
1703         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1704
1705         for (j = 0; j < 2; j++) {
1706                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1707                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1708                                 port_id = fwd_ports_ids[rxp];
1709                                 /**
1710                                 * testpmd can stuck in the below do while loop
1711                                 * if rte_eth_rx_burst() always returns nonzero
1712                                 * packets. So timer is added to exit this loop
1713                                 * after 1sec timer expiry.
1714                                 */
1715                                 prev_tsc = rte_rdtsc();
1716                                 do {
1717                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1718                                                 pkts_burst, MAX_PKT_BURST);
1719                                         for (i = 0; i < nb_rx; i++)
1720                                                 rte_pktmbuf_free(pkts_burst[i]);
1721
1722                                         cur_tsc = rte_rdtsc();
1723                                         diff_tsc = cur_tsc - prev_tsc;
1724                                         timer_tsc += diff_tsc;
1725                                 } while ((nb_rx > 0) &&
1726                                         (timer_tsc < timer_period));
1727                                 timer_tsc = 0;
1728                         }
1729                 }
1730                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1731         }
1732 }
1733
1734 static void
1735 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1736 {
1737         struct fwd_stream **fsm;
1738         streamid_t nb_fs;
1739         streamid_t sm_id;
1740 #ifdef RTE_LIBRTE_BITRATE
1741         uint64_t tics_per_1sec;
1742         uint64_t tics_datum;
1743         uint64_t tics_current;
1744         uint16_t i, cnt_ports;
1745
1746         cnt_ports = nb_ports;
1747         tics_datum = rte_rdtsc();
1748         tics_per_1sec = rte_get_timer_hz();
1749 #endif
1750         fsm = &fwd_streams[fc->stream_idx];
1751         nb_fs = fc->stream_nb;
1752         do {
1753                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1754                         (*pkt_fwd)(fsm[sm_id]);
1755 #ifdef RTE_LIBRTE_BITRATE
1756                 if (bitrate_enabled != 0 &&
1757                                 bitrate_lcore_id == rte_lcore_id()) {
1758                         tics_current = rte_rdtsc();
1759                         if (tics_current - tics_datum >= tics_per_1sec) {
1760                                 /* Periodic bitrate calculation */
1761                                 for (i = 0; i < cnt_ports; i++)
1762                                         rte_stats_bitrate_calc(bitrate_data,
1763                                                 ports_ids[i]);
1764                                 tics_datum = tics_current;
1765                         }
1766                 }
1767 #endif
1768 #ifdef RTE_LIBRTE_LATENCY_STATS
1769                 if (latencystats_enabled != 0 &&
1770                                 latencystats_lcore_id == rte_lcore_id())
1771                         rte_latencystats_update();
1772 #endif
1773
1774         } while (! fc->stopped);
1775 }
1776
1777 static int
1778 start_pkt_forward_on_core(void *fwd_arg)
1779 {
1780         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1781                              cur_fwd_config.fwd_eng->packet_fwd);
1782         return 0;
1783 }
1784
1785 /*
1786  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1787  * Used to start communication flows in network loopback test configurations.
1788  */
1789 static int
1790 run_one_txonly_burst_on_core(void *fwd_arg)
1791 {
1792         struct fwd_lcore *fwd_lc;
1793         struct fwd_lcore tmp_lcore;
1794
1795         fwd_lc = (struct fwd_lcore *) fwd_arg;
1796         tmp_lcore = *fwd_lc;
1797         tmp_lcore.stopped = 1;
1798         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1799         return 0;
1800 }
1801
1802 /*
1803  * Launch packet forwarding:
1804  *     - Setup per-port forwarding context.
1805  *     - launch logical cores with their forwarding configuration.
1806  */
1807 static void
1808 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1809 {
1810         port_fwd_begin_t port_fwd_begin;
1811         unsigned int i;
1812         unsigned int lc_id;
1813         int diag;
1814
1815         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1816         if (port_fwd_begin != NULL) {
1817                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1818                         (*port_fwd_begin)(fwd_ports_ids[i]);
1819         }
1820         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1821                 lc_id = fwd_lcores_cpuids[i];
1822                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1823                         fwd_lcores[i]->stopped = 0;
1824                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1825                                                      fwd_lcores[i], lc_id);
1826                         if (diag != 0)
1827                                 printf("launch lcore %u failed - diag=%d\n",
1828                                        lc_id, diag);
1829                 }
1830         }
1831 }
1832
1833 /*
1834  * Launch packet forwarding configuration.
1835  */
1836 void
1837 start_packet_forwarding(int with_tx_first)
1838 {
1839         port_fwd_begin_t port_fwd_begin;
1840         port_fwd_end_t  port_fwd_end;
1841         struct rte_port *port;
1842         unsigned int i;
1843         portid_t   pt_id;
1844
1845         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1846                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1847
1848         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1849                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1850
1851         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1852                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1853                 (!nb_rxq || !nb_txq))
1854                 rte_exit(EXIT_FAILURE,
1855                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1856                         cur_fwd_eng->fwd_mode_name);
1857
1858         if (all_ports_started() == 0) {
1859                 printf("Not all ports were started\n");
1860                 return;
1861         }
1862         if (test_done == 0) {
1863                 printf("Packet forwarding already started\n");
1864                 return;
1865         }
1866
1867
1868         if(dcb_test) {
1869                 for (i = 0; i < nb_fwd_ports; i++) {
1870                         pt_id = fwd_ports_ids[i];
1871                         port = &ports[pt_id];
1872                         if (!port->dcb_flag) {
1873                                 printf("In DCB mode, all forwarding ports must "
1874                                        "be configured in this mode.\n");
1875                                 return;
1876                         }
1877                 }
1878                 if (nb_fwd_lcores == 1) {
1879                         printf("In DCB mode,the nb forwarding cores "
1880                                "should be larger than 1.\n");
1881                         return;
1882                 }
1883         }
1884         test_done = 0;
1885
1886         fwd_config_setup();
1887
1888         if(!no_flush_rx)
1889                 flush_fwd_rx_queues();
1890
1891         pkt_fwd_config_display(&cur_fwd_config);
1892         rxtx_config_display();
1893
1894         fwd_stats_reset();
1895         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1896                 pt_id = fwd_ports_ids[i];
1897                 port = &ports[pt_id];
1898                 map_port_queue_stats_mapping_registers(pt_id, port);
1899         }
1900         if (with_tx_first) {
1901                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1902                 if (port_fwd_begin != NULL) {
1903                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1904                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1905                 }
1906                 while (with_tx_first--) {
1907                         launch_packet_forwarding(
1908                                         run_one_txonly_burst_on_core);
1909                         rte_eal_mp_wait_lcore();
1910                 }
1911                 port_fwd_end = tx_only_engine.port_fwd_end;
1912                 if (port_fwd_end != NULL) {
1913                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1914                                 (*port_fwd_end)(fwd_ports_ids[i]);
1915                 }
1916         }
1917         launch_packet_forwarding(start_pkt_forward_on_core);
1918 }
1919
1920 void
1921 stop_packet_forwarding(void)
1922 {
1923         port_fwd_end_t port_fwd_end;
1924         lcoreid_t lc_id;
1925         portid_t pt_id;
1926         int i;
1927
1928         if (test_done) {
1929                 printf("Packet forwarding not started\n");
1930                 return;
1931         }
1932         printf("Telling cores to stop...");
1933         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1934                 fwd_lcores[lc_id]->stopped = 1;
1935         printf("\nWaiting for lcores to finish...\n");
1936         rte_eal_mp_wait_lcore();
1937         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1938         if (port_fwd_end != NULL) {
1939                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1940                         pt_id = fwd_ports_ids[i];
1941                         (*port_fwd_end)(pt_id);
1942                 }
1943         }
1944
1945         fwd_stats_display();
1946
1947         printf("\nDone.\n");
1948         test_done = 1;
1949 }
1950
1951 void
1952 dev_set_link_up(portid_t pid)
1953 {
1954         if (rte_eth_dev_set_link_up(pid) < 0)
1955                 printf("\nSet link up fail.\n");
1956 }
1957
1958 void
1959 dev_set_link_down(portid_t pid)
1960 {
1961         if (rte_eth_dev_set_link_down(pid) < 0)
1962                 printf("\nSet link down fail.\n");
1963 }
1964
1965 static int
1966 all_ports_started(void)
1967 {
1968         portid_t pi;
1969         struct rte_port *port;
1970
1971         RTE_ETH_FOREACH_DEV(pi) {
1972                 port = &ports[pi];
1973                 /* Check if there is a port which is not started */
1974                 if ((port->port_status != RTE_PORT_STARTED) &&
1975                         (port->slave_flag == 0))
1976                         return 0;
1977         }
1978
1979         /* No port is not started */
1980         return 1;
1981 }
1982
1983 int
1984 port_is_stopped(portid_t port_id)
1985 {
1986         struct rte_port *port = &ports[port_id];
1987
1988         if ((port->port_status != RTE_PORT_STOPPED) &&
1989             (port->slave_flag == 0))
1990                 return 0;
1991         return 1;
1992 }
1993
1994 int
1995 all_ports_stopped(void)
1996 {
1997         portid_t pi;
1998
1999         RTE_ETH_FOREACH_DEV(pi) {
2000                 if (!port_is_stopped(pi))
2001                         return 0;
2002         }
2003
2004         return 1;
2005 }
2006
2007 int
2008 port_is_started(portid_t port_id)
2009 {
2010         if (port_id_is_invalid(port_id, ENABLED_WARN))
2011                 return 0;
2012
2013         if (ports[port_id].port_status != RTE_PORT_STARTED)
2014                 return 0;
2015
2016         return 1;
2017 }
2018
2019 int
2020 start_port(portid_t pid)
2021 {
2022         int diag, need_check_link_status = -1;
2023         portid_t pi;
2024         queueid_t qi;
2025         struct rte_port *port;
2026         struct rte_ether_addr mac_addr;
2027
2028         if (port_id_is_invalid(pid, ENABLED_WARN))
2029                 return 0;
2030
2031         if(dcb_config)
2032                 dcb_test = 1;
2033         RTE_ETH_FOREACH_DEV(pi) {
2034                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2035                         continue;
2036
2037                 need_check_link_status = 0;
2038                 port = &ports[pi];
2039                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2040                                                  RTE_PORT_HANDLING) == 0) {
2041                         printf("Port %d is now not stopped\n", pi);
2042                         continue;
2043                 }
2044
2045                 if (port->need_reconfig > 0) {
2046                         port->need_reconfig = 0;
2047
2048                         if (flow_isolate_all) {
2049                                 int ret = port_flow_isolate(pi, 1);
2050                                 if (ret) {
2051                                         printf("Failed to apply isolated"
2052                                                " mode on port %d\n", pi);
2053                                         return -1;
2054                                 }
2055                         }
2056                         configure_rxtx_dump_callbacks(0);
2057                         printf("Configuring Port %d (socket %u)\n", pi,
2058                                         port->socket_id);
2059                         /* configure port */
2060                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
2061                                                 &(port->dev_conf));
2062                         if (diag != 0) {
2063                                 if (rte_atomic16_cmpset(&(port->port_status),
2064                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2065                                         printf("Port %d can not be set back "
2066                                                         "to stopped\n", pi);
2067                                 printf("Fail to configure port %d\n", pi);
2068                                 /* try to reconfigure port next time */
2069                                 port->need_reconfig = 1;
2070                                 return -1;
2071                         }
2072                 }
2073                 if (port->need_reconfig_queues > 0) {
2074                         port->need_reconfig_queues = 0;
2075                         /* setup tx queues */
2076                         for (qi = 0; qi < nb_txq; qi++) {
2077                                 if ((numa_support) &&
2078                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2079                                         diag = rte_eth_tx_queue_setup(pi, qi,
2080                                                 port->nb_tx_desc[qi],
2081                                                 txring_numa[pi],
2082                                                 &(port->tx_conf[qi]));
2083                                 else
2084                                         diag = rte_eth_tx_queue_setup(pi, qi,
2085                                                 port->nb_tx_desc[qi],
2086                                                 port->socket_id,
2087                                                 &(port->tx_conf[qi]));
2088
2089                                 if (diag == 0)
2090                                         continue;
2091
2092                                 /* Fail to setup tx queue, return */
2093                                 if (rte_atomic16_cmpset(&(port->port_status),
2094                                                         RTE_PORT_HANDLING,
2095                                                         RTE_PORT_STOPPED) == 0)
2096                                         printf("Port %d can not be set back "
2097                                                         "to stopped\n", pi);
2098                                 printf("Fail to configure port %d tx queues\n",
2099                                        pi);
2100                                 /* try to reconfigure queues next time */
2101                                 port->need_reconfig_queues = 1;
2102                                 return -1;
2103                         }
2104                         for (qi = 0; qi < nb_rxq; qi++) {
2105                                 /* setup rx queues */
2106                                 if ((numa_support) &&
2107                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2108                                         struct rte_mempool * mp =
2109                                                 mbuf_pool_find(rxring_numa[pi]);
2110                                         if (mp == NULL) {
2111                                                 printf("Failed to setup RX queue:"
2112                                                         "No mempool allocation"
2113                                                         " on the socket %d\n",
2114                                                         rxring_numa[pi]);
2115                                                 return -1;
2116                                         }
2117
2118                                         diag = rte_eth_rx_queue_setup(pi, qi,
2119                                              port->nb_rx_desc[qi],
2120                                              rxring_numa[pi],
2121                                              &(port->rx_conf[qi]),
2122                                              mp);
2123                                 } else {
2124                                         struct rte_mempool *mp =
2125                                                 mbuf_pool_find(port->socket_id);
2126                                         if (mp == NULL) {
2127                                                 printf("Failed to setup RX queue:"
2128                                                         "No mempool allocation"
2129                                                         " on the socket %d\n",
2130                                                         port->socket_id);
2131                                                 return -1;
2132                                         }
2133                                         diag = rte_eth_rx_queue_setup(pi, qi,
2134                                              port->nb_rx_desc[qi],
2135                                              port->socket_id,
2136                                              &(port->rx_conf[qi]),
2137                                              mp);
2138                                 }
2139                                 if (diag == 0)
2140                                         continue;
2141
2142                                 /* Fail to setup rx queue, return */
2143                                 if (rte_atomic16_cmpset(&(port->port_status),
2144                                                         RTE_PORT_HANDLING,
2145                                                         RTE_PORT_STOPPED) == 0)
2146                                         printf("Port %d can not be set back "
2147                                                         "to stopped\n", pi);
2148                                 printf("Fail to configure port %d rx queues\n",
2149                                        pi);
2150                                 /* try to reconfigure queues next time */
2151                                 port->need_reconfig_queues = 1;
2152                                 return -1;
2153                         }
2154                 }
2155                 configure_rxtx_dump_callbacks(verbose_level);
2156                 /* start port */
2157                 if (rte_eth_dev_start(pi) < 0) {
2158                         printf("Fail to start port %d\n", pi);
2159
2160                         /* Fail to setup rx queue, return */
2161                         if (rte_atomic16_cmpset(&(port->port_status),
2162                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2163                                 printf("Port %d can not be set back to "
2164                                                         "stopped\n", pi);
2165                         continue;
2166                 }
2167
2168                 if (rte_atomic16_cmpset(&(port->port_status),
2169                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2170                         printf("Port %d can not be set into started\n", pi);
2171
2172                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2173                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2174                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2175                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2176                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2177
2178                 /* at least one port started, need checking link status */
2179                 need_check_link_status = 1;
2180         }
2181
2182         if (need_check_link_status == 1 && !no_link_check)
2183                 check_all_ports_link_status(RTE_PORT_ALL);
2184         else if (need_check_link_status == 0)
2185                 printf("Please stop the ports first\n");
2186
2187         printf("Done\n");
2188         return 0;
2189 }
2190
2191 void
2192 stop_port(portid_t pid)
2193 {
2194         portid_t pi;
2195         struct rte_port *port;
2196         int need_check_link_status = 0;
2197
2198         if (dcb_test) {
2199                 dcb_test = 0;
2200                 dcb_config = 0;
2201         }
2202
2203         if (port_id_is_invalid(pid, ENABLED_WARN))
2204                 return;
2205
2206         printf("Stopping ports...\n");
2207
2208         RTE_ETH_FOREACH_DEV(pi) {
2209                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2210                         continue;
2211
2212                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2213                         printf("Please remove port %d from forwarding configuration.\n", pi);
2214                         continue;
2215                 }
2216
2217                 if (port_is_bonding_slave(pi)) {
2218                         printf("Please remove port %d from bonded device.\n", pi);
2219                         continue;
2220                 }
2221
2222                 port = &ports[pi];
2223                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2224                                                 RTE_PORT_HANDLING) == 0)
2225                         continue;
2226
2227                 rte_eth_dev_stop(pi);
2228
2229                 if (rte_atomic16_cmpset(&(port->port_status),
2230                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2231                         printf("Port %d can not be set into stopped\n", pi);
2232                 need_check_link_status = 1;
2233         }
2234         if (need_check_link_status && !no_link_check)
2235                 check_all_ports_link_status(RTE_PORT_ALL);
2236
2237         printf("Done\n");
2238 }
2239
2240 static void
2241 remove_invalid_ports_in(portid_t *array, portid_t *total)
2242 {
2243         portid_t i;
2244         portid_t new_total = 0;
2245
2246         for (i = 0; i < *total; i++)
2247                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2248                         array[new_total] = array[i];
2249                         new_total++;
2250                 }
2251         *total = new_total;
2252 }
2253
2254 static void
2255 remove_invalid_ports(void)
2256 {
2257         remove_invalid_ports_in(ports_ids, &nb_ports);
2258         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2259         nb_cfg_ports = nb_fwd_ports;
2260 }
2261
2262 void
2263 close_port(portid_t pid)
2264 {
2265         portid_t pi;
2266         struct rte_port *port;
2267
2268         if (port_id_is_invalid(pid, ENABLED_WARN))
2269                 return;
2270
2271         printf("Closing ports...\n");
2272
2273         RTE_ETH_FOREACH_DEV(pi) {
2274                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2275                         continue;
2276
2277                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2278                         printf("Please remove port %d from forwarding configuration.\n", pi);
2279                         continue;
2280                 }
2281
2282                 if (port_is_bonding_slave(pi)) {
2283                         printf("Please remove port %d from bonded device.\n", pi);
2284                         continue;
2285                 }
2286
2287                 port = &ports[pi];
2288                 if (rte_atomic16_cmpset(&(port->port_status),
2289                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2290                         printf("Port %d is already closed\n", pi);
2291                         continue;
2292                 }
2293
2294                 if (rte_atomic16_cmpset(&(port->port_status),
2295                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2296                         printf("Port %d is now not stopped\n", pi);
2297                         continue;
2298                 }
2299
2300                 if (port->flow_list)
2301                         port_flow_flush(pi);
2302                 rte_eth_dev_close(pi);
2303
2304                 remove_invalid_ports();
2305
2306                 if (rte_atomic16_cmpset(&(port->port_status),
2307                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2308                         printf("Port %d cannot be set to closed\n", pi);
2309         }
2310
2311         printf("Done\n");
2312 }
2313
2314 void
2315 reset_port(portid_t pid)
2316 {
2317         int diag;
2318         portid_t pi;
2319         struct rte_port *port;
2320
2321         if (port_id_is_invalid(pid, ENABLED_WARN))
2322                 return;
2323
2324         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2325                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2326                 printf("Can not reset port(s), please stop port(s) first.\n");
2327                 return;
2328         }
2329
2330         printf("Resetting ports...\n");
2331
2332         RTE_ETH_FOREACH_DEV(pi) {
2333                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2334                         continue;
2335
2336                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2337                         printf("Please remove port %d from forwarding "
2338                                "configuration.\n", pi);
2339                         continue;
2340                 }
2341
2342                 if (port_is_bonding_slave(pi)) {
2343                         printf("Please remove port %d from bonded device.\n",
2344                                pi);
2345                         continue;
2346                 }
2347
2348                 diag = rte_eth_dev_reset(pi);
2349                 if (diag == 0) {
2350                         port = &ports[pi];
2351                         port->need_reconfig = 1;
2352                         port->need_reconfig_queues = 1;
2353                 } else {
2354                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2355                 }
2356         }
2357
2358         printf("Done\n");
2359 }
2360
2361 void
2362 attach_port(char *identifier)
2363 {
2364         portid_t pi;
2365         struct rte_dev_iterator iterator;
2366
2367         printf("Attaching a new port...\n");
2368
2369         if (identifier == NULL) {
2370                 printf("Invalid parameters are specified\n");
2371                 return;
2372         }
2373
2374         if (rte_dev_probe(identifier) < 0) {
2375                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2376                 return;
2377         }
2378
2379         /* first attach mode: event */
2380         if (setup_on_probe_event) {
2381                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2382                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2383                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2384                                         ports[pi].need_setup != 0)
2385                                 setup_attached_port(pi);
2386                 return;
2387         }
2388
2389         /* second attach mode: iterator */
2390         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2391                 /* setup ports matching the devargs used for probing */
2392                 if (port_is_forwarding(pi))
2393                         continue; /* port was already attached before */
2394                 setup_attached_port(pi);
2395         }
2396 }
2397
2398 static void
2399 setup_attached_port(portid_t pi)
2400 {
2401         unsigned int socket_id;
2402         int ret;
2403
2404         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2405         /* if socket_id is invalid, set to the first available socket. */
2406         if (check_socket_id(socket_id) < 0)
2407                 socket_id = socket_ids[0];
2408         reconfig(pi, socket_id);
2409         ret = rte_eth_promiscuous_enable(pi);
2410         if (ret != 0)
2411                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2412                         pi, rte_strerror(-ret));
2413
2414         ports_ids[nb_ports++] = pi;
2415         fwd_ports_ids[nb_fwd_ports++] = pi;
2416         nb_cfg_ports = nb_fwd_ports;
2417         ports[pi].need_setup = 0;
2418         ports[pi].port_status = RTE_PORT_STOPPED;
2419
2420         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2421         printf("Done\n");
2422 }
2423
2424 void
2425 detach_port_device(portid_t port_id)
2426 {
2427         struct rte_device *dev;
2428         portid_t sibling;
2429
2430         printf("Removing a device...\n");
2431
2432         dev = rte_eth_devices[port_id].device;
2433         if (dev == NULL) {
2434                 printf("Device already removed\n");
2435                 return;
2436         }
2437
2438         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2439                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2440                         printf("Port not stopped\n");
2441                         return;
2442                 }
2443                 printf("Port was not closed\n");
2444                 if (ports[port_id].flow_list)
2445                         port_flow_flush(port_id);
2446         }
2447
2448         if (rte_dev_remove(dev) < 0) {
2449                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2450                 return;
2451         }
2452         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2453                 /* reset mapping between old ports and removed device */
2454                 rte_eth_devices[sibling].device = NULL;
2455                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2456                         /* sibling ports are forced to be closed */
2457                         ports[sibling].port_status = RTE_PORT_CLOSED;
2458                         printf("Port %u is closed\n", sibling);
2459                 }
2460         }
2461
2462         remove_invalid_ports();
2463
2464         printf("Device of port %u is detached\n", port_id);
2465         printf("Now total ports is %d\n", nb_ports);
2466         printf("Done\n");
2467         return;
2468 }
2469
2470 void
2471 detach_device(char *identifier)
2472 {
2473         struct rte_dev_iterator iterator;
2474         struct rte_devargs da;
2475         portid_t port_id;
2476
2477         printf("Removing a device...\n");
2478
2479         memset(&da, 0, sizeof(da));
2480         if (rte_devargs_parsef(&da, "%s", identifier)) {
2481                 printf("cannot parse identifier\n");
2482                 if (da.args)
2483                         free(da.args);
2484                 return;
2485         }
2486
2487         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2488                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2489                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2490                                 printf("Port %u not stopped\n", port_id);
2491                                 return;
2492                         }
2493
2494                         /* sibling ports are forced to be closed */
2495                         if (ports[port_id].flow_list)
2496                                 port_flow_flush(port_id);
2497                         ports[port_id].port_status = RTE_PORT_CLOSED;
2498                         printf("Port %u is now closed\n", port_id);
2499                 }
2500         }
2501
2502         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2503                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2504                             da.name, da.bus->name);
2505                 return;
2506         }
2507
2508         remove_invalid_ports();
2509
2510         printf("Device %s is detached\n", identifier);
2511         printf("Now total ports is %d\n", nb_ports);
2512         printf("Done\n");
2513 }
2514
2515 void
2516 pmd_test_exit(void)
2517 {
2518         portid_t pt_id;
2519         int ret;
2520         int i;
2521
2522         if (test_done == 0)
2523                 stop_packet_forwarding();
2524
2525         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2526                 if (mempools[i]) {
2527                         if (mp_alloc_type == MP_ALLOC_ANON)
2528                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2529                                                      NULL);
2530                 }
2531         }
2532         if (ports != NULL) {
2533                 no_link_check = 1;
2534                 RTE_ETH_FOREACH_DEV(pt_id) {
2535                         printf("\nStopping port %d...\n", pt_id);
2536                         fflush(stdout);
2537                         stop_port(pt_id);
2538                 }
2539                 RTE_ETH_FOREACH_DEV(pt_id) {
2540                         printf("\nShutting down port %d...\n", pt_id);
2541                         fflush(stdout);
2542                         close_port(pt_id);
2543                 }
2544         }
2545
2546         if (hot_plug) {
2547                 ret = rte_dev_event_monitor_stop();
2548                 if (ret) {
2549                         RTE_LOG(ERR, EAL,
2550                                 "fail to stop device event monitor.");
2551                         return;
2552                 }
2553
2554                 ret = rte_dev_event_callback_unregister(NULL,
2555                         dev_event_callback, NULL);
2556                 if (ret < 0) {
2557                         RTE_LOG(ERR, EAL,
2558                                 "fail to unregister device event callback.\n");
2559                         return;
2560                 }
2561
2562                 ret = rte_dev_hotplug_handle_disable();
2563                 if (ret) {
2564                         RTE_LOG(ERR, EAL,
2565                                 "fail to disable hotplug handling.\n");
2566                         return;
2567                 }
2568         }
2569         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2570                 if (mempools[i])
2571                         rte_mempool_free(mempools[i]);
2572         }
2573
2574         printf("\nBye...\n");
2575 }
2576
2577 typedef void (*cmd_func_t)(void);
2578 struct pmd_test_command {
2579         const char *cmd_name;
2580         cmd_func_t cmd_func;
2581 };
2582
2583 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2584
2585 /* Check the link status of all ports in up to 9s, and print them finally */
2586 static void
2587 check_all_ports_link_status(uint32_t port_mask)
2588 {
2589 #define CHECK_INTERVAL 100 /* 100ms */
2590 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2591         portid_t portid;
2592         uint8_t count, all_ports_up, print_flag = 0;
2593         struct rte_eth_link link;
2594         int ret;
2595
2596         printf("Checking link statuses...\n");
2597         fflush(stdout);
2598         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2599                 all_ports_up = 1;
2600                 RTE_ETH_FOREACH_DEV(portid) {
2601                         if ((port_mask & (1 << portid)) == 0)
2602                                 continue;
2603                         memset(&link, 0, sizeof(link));
2604                         ret = rte_eth_link_get_nowait(portid, &link);
2605                         if (ret < 0) {
2606                                 all_ports_up = 0;
2607                                 if (print_flag == 1)
2608                                         printf("Port %u link get failed: %s\n",
2609                                                 portid, rte_strerror(-ret));
2610                                 continue;
2611                         }
2612                         /* print link status if flag set */
2613                         if (print_flag == 1) {
2614                                 if (link.link_status)
2615                                         printf(
2616                                         "Port%d Link Up. speed %u Mbps- %s\n",
2617                                         portid, link.link_speed,
2618                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2619                                         ("full-duplex") : ("half-duplex\n"));
2620                                 else
2621                                         printf("Port %d Link Down\n", portid);
2622                                 continue;
2623                         }
2624                         /* clear all_ports_up flag if any link down */
2625                         if (link.link_status == ETH_LINK_DOWN) {
2626                                 all_ports_up = 0;
2627                                 break;
2628                         }
2629                 }
2630                 /* after finally printing all link status, get out */
2631                 if (print_flag == 1)
2632                         break;
2633
2634                 if (all_ports_up == 0) {
2635                         fflush(stdout);
2636                         rte_delay_ms(CHECK_INTERVAL);
2637                 }
2638
2639                 /* set the print_flag if all ports up or timeout */
2640                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2641                         print_flag = 1;
2642                 }
2643
2644                 if (lsc_interrupt)
2645                         break;
2646         }
2647 }
2648
2649 /*
2650  * This callback is for remove a port for a device. It has limitation because
2651  * it is not for multiple port removal for a device.
2652  * TODO: the device detach invoke will plan to be removed from user side to
2653  * eal. And convert all PMDs to free port resources on ether device closing.
2654  */
2655 static void
2656 rmv_port_callback(void *arg)
2657 {
2658         int need_to_start = 0;
2659         int org_no_link_check = no_link_check;
2660         portid_t port_id = (intptr_t)arg;
2661
2662         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2663
2664         if (!test_done && port_is_forwarding(port_id)) {
2665                 need_to_start = 1;
2666                 stop_packet_forwarding();
2667         }
2668         no_link_check = 1;
2669         stop_port(port_id);
2670         no_link_check = org_no_link_check;
2671         close_port(port_id);
2672         detach_port_device(port_id);
2673         if (need_to_start)
2674                 start_packet_forwarding(0);
2675 }
2676
2677 /* This function is used by the interrupt thread */
2678 static int
2679 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2680                   void *ret_param)
2681 {
2682         RTE_SET_USED(param);
2683         RTE_SET_USED(ret_param);
2684
2685         if (type >= RTE_ETH_EVENT_MAX) {
2686                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2687                         port_id, __func__, type);
2688                 fflush(stderr);
2689         } else if (event_print_mask & (UINT32_C(1) << type)) {
2690                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2691                         eth_event_desc[type]);
2692                 fflush(stdout);
2693         }
2694
2695         switch (type) {
2696         case RTE_ETH_EVENT_NEW:
2697                 ports[port_id].need_setup = 1;
2698                 ports[port_id].port_status = RTE_PORT_HANDLING;
2699                 break;
2700         case RTE_ETH_EVENT_INTR_RMV:
2701                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2702                         break;
2703                 if (rte_eal_alarm_set(100000,
2704                                 rmv_port_callback, (void *)(intptr_t)port_id))
2705                         fprintf(stderr, "Could not set up deferred device removal\n");
2706                 break;
2707         default:
2708                 break;
2709         }
2710         return 0;
2711 }
2712
2713 static int
2714 register_eth_event_callback(void)
2715 {
2716         int ret;
2717         enum rte_eth_event_type event;
2718
2719         for (event = RTE_ETH_EVENT_UNKNOWN;
2720                         event < RTE_ETH_EVENT_MAX; event++) {
2721                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2722                                 event,
2723                                 eth_event_callback,
2724                                 NULL);
2725                 if (ret != 0) {
2726                         TESTPMD_LOG(ERR, "Failed to register callback for "
2727                                         "%s event\n", eth_event_desc[event]);
2728                         return -1;
2729                 }
2730         }
2731
2732         return 0;
2733 }
2734
2735 /* This function is used by the interrupt thread */
2736 static void
2737 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2738                              __rte_unused void *arg)
2739 {
2740         uint16_t port_id;
2741         int ret;
2742
2743         if (type >= RTE_DEV_EVENT_MAX) {
2744                 fprintf(stderr, "%s called upon invalid event %d\n",
2745                         __func__, type);
2746                 fflush(stderr);
2747         }
2748
2749         switch (type) {
2750         case RTE_DEV_EVENT_REMOVE:
2751                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2752                         device_name);
2753                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2754                 if (ret) {
2755                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2756                                 device_name);
2757                         return;
2758                 }
2759                 /*
2760                  * Because the user's callback is invoked in eal interrupt
2761                  * callback, the interrupt callback need to be finished before
2762                  * it can be unregistered when detaching device. So finish
2763                  * callback soon and use a deferred removal to detach device
2764                  * is need. It is a workaround, once the device detaching be
2765                  * moved into the eal in the future, the deferred removal could
2766                  * be deleted.
2767                  */
2768                 if (rte_eal_alarm_set(100000,
2769                                 rmv_port_callback, (void *)(intptr_t)port_id))
2770                         RTE_LOG(ERR, EAL,
2771                                 "Could not set up deferred device removal\n");
2772                 break;
2773         case RTE_DEV_EVENT_ADD:
2774                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2775                         device_name);
2776                 /* TODO: After finish kernel driver binding,
2777                  * begin to attach port.
2778                  */
2779                 break;
2780         default:
2781                 break;
2782         }
2783 }
2784
2785 static int
2786 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2787 {
2788         uint16_t i;
2789         int diag;
2790         uint8_t mapping_found = 0;
2791
2792         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2793                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2794                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2795                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2796                                         tx_queue_stats_mappings[i].queue_id,
2797                                         tx_queue_stats_mappings[i].stats_counter_id);
2798                         if (diag != 0)
2799                                 return diag;
2800                         mapping_found = 1;
2801                 }
2802         }
2803         if (mapping_found)
2804                 port->tx_queue_stats_mapping_enabled = 1;
2805         return 0;
2806 }
2807
2808 static int
2809 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2810 {
2811         uint16_t i;
2812         int diag;
2813         uint8_t mapping_found = 0;
2814
2815         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2816                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2817                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2818                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2819                                         rx_queue_stats_mappings[i].queue_id,
2820                                         rx_queue_stats_mappings[i].stats_counter_id);
2821                         if (diag != 0)
2822                                 return diag;
2823                         mapping_found = 1;
2824                 }
2825         }
2826         if (mapping_found)
2827                 port->rx_queue_stats_mapping_enabled = 1;
2828         return 0;
2829 }
2830
2831 static void
2832 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2833 {
2834         int diag = 0;
2835
2836         diag = set_tx_queue_stats_mapping_registers(pi, port);
2837         if (diag != 0) {
2838                 if (diag == -ENOTSUP) {
2839                         port->tx_queue_stats_mapping_enabled = 0;
2840                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2841                 }
2842                 else
2843                         rte_exit(EXIT_FAILURE,
2844                                         "set_tx_queue_stats_mapping_registers "
2845                                         "failed for port id=%d diag=%d\n",
2846                                         pi, diag);
2847         }
2848
2849         diag = set_rx_queue_stats_mapping_registers(pi, port);
2850         if (diag != 0) {
2851                 if (diag == -ENOTSUP) {
2852                         port->rx_queue_stats_mapping_enabled = 0;
2853                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2854                 }
2855                 else
2856                         rte_exit(EXIT_FAILURE,
2857                                         "set_rx_queue_stats_mapping_registers "
2858                                         "failed for port id=%d diag=%d\n",
2859                                         pi, diag);
2860         }
2861 }
2862
2863 static void
2864 rxtx_port_config(struct rte_port *port)
2865 {
2866         uint16_t qid;
2867         uint64_t offloads;
2868
2869         for (qid = 0; qid < nb_rxq; qid++) {
2870                 offloads = port->rx_conf[qid].offloads;
2871                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2872                 if (offloads != 0)
2873                         port->rx_conf[qid].offloads = offloads;
2874
2875                 /* Check if any Rx parameters have been passed */
2876                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2877                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2878
2879                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2880                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2881
2882                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2883                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2884
2885                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2886                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2887
2888                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2889                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2890
2891                 port->nb_rx_desc[qid] = nb_rxd;
2892         }
2893
2894         for (qid = 0; qid < nb_txq; qid++) {
2895                 offloads = port->tx_conf[qid].offloads;
2896                 port->tx_conf[qid] = port->dev_info.default_txconf;
2897                 if (offloads != 0)
2898                         port->tx_conf[qid].offloads = offloads;
2899
2900                 /* Check if any Tx parameters have been passed */
2901                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2902                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2903
2904                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2905                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2906
2907                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2908                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2909
2910                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2911                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2912
2913                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2914                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2915
2916                 port->nb_tx_desc[qid] = nb_txd;
2917         }
2918 }
2919
2920 void
2921 init_port_config(void)
2922 {
2923         portid_t pid;
2924         struct rte_port *port;
2925         int ret;
2926
2927         RTE_ETH_FOREACH_DEV(pid) {
2928                 port = &ports[pid];
2929                 port->dev_conf.fdir_conf = fdir_conf;
2930
2931                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
2932                 if (ret != 0)
2933                         return;
2934
2935                 if (nb_rxq > 1) {
2936                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2937                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2938                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2939                 } else {
2940                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2941                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2942                 }
2943
2944                 if (port->dcb_flag == 0) {
2945                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2946                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2947                         else
2948                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2949                 }
2950
2951                 rxtx_port_config(port);
2952
2953                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
2954                 if (ret != 0)
2955                         return;
2956
2957                 map_port_queue_stats_mapping_registers(pid, port);
2958 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2959                 rte_pmd_ixgbe_bypass_init(pid);
2960 #endif
2961
2962                 if (lsc_interrupt &&
2963                     (rte_eth_devices[pid].data->dev_flags &
2964                      RTE_ETH_DEV_INTR_LSC))
2965                         port->dev_conf.intr_conf.lsc = 1;
2966                 if (rmv_interrupt &&
2967                     (rte_eth_devices[pid].data->dev_flags &
2968                      RTE_ETH_DEV_INTR_RMV))
2969                         port->dev_conf.intr_conf.rmv = 1;
2970         }
2971 }
2972
2973 void set_port_slave_flag(portid_t slave_pid)
2974 {
2975         struct rte_port *port;
2976
2977         port = &ports[slave_pid];
2978         port->slave_flag = 1;
2979 }
2980
2981 void clear_port_slave_flag(portid_t slave_pid)
2982 {
2983         struct rte_port *port;
2984
2985         port = &ports[slave_pid];
2986         port->slave_flag = 0;
2987 }
2988
2989 uint8_t port_is_bonding_slave(portid_t slave_pid)
2990 {
2991         struct rte_port *port;
2992
2993         port = &ports[slave_pid];
2994         if ((rte_eth_devices[slave_pid].data->dev_flags &
2995             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
2996                 return 1;
2997         return 0;
2998 }
2999
3000 const uint16_t vlan_tags[] = {
3001                 0,  1,  2,  3,  4,  5,  6,  7,
3002                 8,  9, 10, 11,  12, 13, 14, 15,
3003                 16, 17, 18, 19, 20, 21, 22, 23,
3004                 24, 25, 26, 27, 28, 29, 30, 31
3005 };
3006
3007 static  int
3008 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3009                  enum dcb_mode_enable dcb_mode,
3010                  enum rte_eth_nb_tcs num_tcs,
3011                  uint8_t pfc_en)
3012 {
3013         uint8_t i;
3014         int32_t rc;
3015         struct rte_eth_rss_conf rss_conf;
3016
3017         /*
3018          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3019          * given above, and the number of traffic classes available for use.
3020          */
3021         if (dcb_mode == DCB_VT_ENABLED) {
3022                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3023                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3024                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3025                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3026
3027                 /* VMDQ+DCB RX and TX configurations */
3028                 vmdq_rx_conf->enable_default_pool = 0;
3029                 vmdq_rx_conf->default_pool = 0;
3030                 vmdq_rx_conf->nb_queue_pools =
3031                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3032                 vmdq_tx_conf->nb_queue_pools =
3033                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3034
3035                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3036                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3037                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3038                         vmdq_rx_conf->pool_map[i].pools =
3039                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3040                 }
3041                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3042                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3043                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3044                 }
3045
3046                 /* set DCB mode of RX and TX of multiple queues */
3047                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
3048                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3049         } else {
3050                 struct rte_eth_dcb_rx_conf *rx_conf =
3051                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3052                 struct rte_eth_dcb_tx_conf *tx_conf =
3053                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3054
3055                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3056                 if (rc != 0)
3057                         return rc;
3058
3059                 rx_conf->nb_tcs = num_tcs;
3060                 tx_conf->nb_tcs = num_tcs;
3061
3062                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3063                         rx_conf->dcb_tc[i] = i % num_tcs;
3064                         tx_conf->dcb_tc[i] = i % num_tcs;
3065                 }
3066
3067                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
3068                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3069                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3070         }
3071
3072         if (pfc_en)
3073                 eth_conf->dcb_capability_en =
3074                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3075         else
3076                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3077
3078         return 0;
3079 }
3080
3081 int
3082 init_port_dcb_config(portid_t pid,
3083                      enum dcb_mode_enable dcb_mode,
3084                      enum rte_eth_nb_tcs num_tcs,
3085                      uint8_t pfc_en)
3086 {
3087         struct rte_eth_conf port_conf;
3088         struct rte_port *rte_port;
3089         int retval;
3090         uint16_t i;
3091
3092         rte_port = &ports[pid];
3093
3094         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3095         /* Enter DCB configuration status */
3096         dcb_config = 1;
3097
3098         port_conf.rxmode = rte_port->dev_conf.rxmode;
3099         port_conf.txmode = rte_port->dev_conf.txmode;
3100
3101         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3102         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3103         if (retval < 0)
3104                 return retval;
3105         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3106
3107         /* re-configure the device . */
3108         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3109         if (retval < 0)
3110                 return retval;
3111
3112         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3113         if (retval != 0)
3114                 return retval;
3115
3116         /* If dev_info.vmdq_pool_base is greater than 0,
3117          * the queue id of vmdq pools is started after pf queues.
3118          */
3119         if (dcb_mode == DCB_VT_ENABLED &&
3120             rte_port->dev_info.vmdq_pool_base > 0) {
3121                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3122                         " for port %d.", pid);
3123                 return -1;
3124         }
3125
3126         /* Assume the ports in testpmd have the same dcb capability
3127          * and has the same number of rxq and txq in dcb mode
3128          */
3129         if (dcb_mode == DCB_VT_ENABLED) {
3130                 if (rte_port->dev_info.max_vfs > 0) {
3131                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3132                         nb_txq = rte_port->dev_info.nb_tx_queues;
3133                 } else {
3134                         nb_rxq = rte_port->dev_info.max_rx_queues;
3135                         nb_txq = rte_port->dev_info.max_tx_queues;
3136                 }
3137         } else {
3138                 /*if vt is disabled, use all pf queues */
3139                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3140                         nb_rxq = rte_port->dev_info.max_rx_queues;
3141                         nb_txq = rte_port->dev_info.max_tx_queues;
3142                 } else {
3143                         nb_rxq = (queueid_t)num_tcs;
3144                         nb_txq = (queueid_t)num_tcs;
3145
3146                 }
3147         }
3148         rx_free_thresh = 64;
3149
3150         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3151
3152         rxtx_port_config(rte_port);
3153         /* VLAN filter */
3154         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3155         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3156                 rx_vft_set(pid, vlan_tags[i], 1);
3157
3158         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3159         if (retval != 0)
3160                 return retval;
3161
3162         map_port_queue_stats_mapping_registers(pid, rte_port);
3163
3164         rte_port->dcb_flag = 1;
3165
3166         return 0;
3167 }
3168
3169 static void
3170 init_port(void)
3171 {
3172         /* Configuration of Ethernet ports. */
3173         ports = rte_zmalloc("testpmd: ports",
3174                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3175                             RTE_CACHE_LINE_SIZE);
3176         if (ports == NULL) {
3177                 rte_exit(EXIT_FAILURE,
3178                                 "rte_zmalloc(%d struct rte_port) failed\n",
3179                                 RTE_MAX_ETHPORTS);
3180         }
3181
3182         /* Initialize ports NUMA structures */
3183         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3184         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3185         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3186 }
3187
3188 static void
3189 force_quit(void)
3190 {
3191         pmd_test_exit();
3192         prompt_exit();
3193 }
3194
3195 static void
3196 print_stats(void)
3197 {
3198         uint8_t i;
3199         const char clr[] = { 27, '[', '2', 'J', '\0' };
3200         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3201
3202         /* Clear screen and move to top left */
3203         printf("%s%s", clr, top_left);
3204
3205         printf("\nPort statistics ====================================");
3206         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3207                 nic_stats_display(fwd_ports_ids[i]);
3208
3209         fflush(stdout);
3210 }
3211
3212 static void
3213 signal_handler(int signum)
3214 {
3215         if (signum == SIGINT || signum == SIGTERM) {
3216                 printf("\nSignal %d received, preparing to exit...\n",
3217                                 signum);
3218 #ifdef RTE_LIBRTE_PDUMP
3219                 /* uninitialize packet capture framework */
3220                 rte_pdump_uninit();
3221 #endif
3222 #ifdef RTE_LIBRTE_LATENCY_STATS
3223                 if (latencystats_enabled != 0)
3224                         rte_latencystats_uninit();
3225 #endif
3226                 force_quit();
3227                 /* Set flag to indicate the force termination. */
3228                 f_quit = 1;
3229                 /* exit with the expected status */
3230                 signal(signum, SIG_DFL);
3231                 kill(getpid(), signum);
3232         }
3233 }
3234
3235 int
3236 main(int argc, char** argv)
3237 {
3238         int diag;
3239         portid_t port_id;
3240         uint16_t count;
3241         int ret;
3242
3243         signal(SIGINT, signal_handler);
3244         signal(SIGTERM, signal_handler);
3245
3246         testpmd_logtype = rte_log_register("testpmd");
3247         if (testpmd_logtype < 0)
3248                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3249         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3250
3251         diag = rte_eal_init(argc, argv);
3252         if (diag < 0)
3253                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3254                          rte_strerror(rte_errno));
3255
3256         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3257                 rte_exit(EXIT_FAILURE,
3258                          "Secondary process type not supported.\n");
3259
3260         ret = register_eth_event_callback();
3261         if (ret != 0)
3262                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3263
3264 #ifdef RTE_LIBRTE_PDUMP
3265         /* initialize packet capture framework */
3266         rte_pdump_init();
3267 #endif
3268
3269         count = 0;
3270         RTE_ETH_FOREACH_DEV(port_id) {
3271                 ports_ids[count] = port_id;
3272                 count++;
3273         }
3274         nb_ports = (portid_t) count;
3275         if (nb_ports == 0)
3276                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3277
3278         /* allocate port structures, and init them */
3279         init_port();
3280
3281         set_def_fwd_config();
3282         if (nb_lcores == 0)
3283                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3284                          "Check the core mask argument\n");
3285
3286         /* Bitrate/latency stats disabled by default */
3287 #ifdef RTE_LIBRTE_BITRATE
3288         bitrate_enabled = 0;
3289 #endif
3290 #ifdef RTE_LIBRTE_LATENCY_STATS
3291         latencystats_enabled = 0;
3292 #endif
3293
3294         /* on FreeBSD, mlockall() is disabled by default */
3295 #ifdef RTE_EXEC_ENV_FREEBSD
3296         do_mlockall = 0;
3297 #else
3298         do_mlockall = 1;
3299 #endif
3300
3301         argc -= diag;
3302         argv += diag;
3303         if (argc > 1)
3304                 launch_args_parse(argc, argv);
3305
3306         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3307                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3308                         strerror(errno));
3309         }
3310
3311         if (tx_first && interactive)
3312                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3313                                 "interactive mode.\n");
3314
3315         if (tx_first && lsc_interrupt) {
3316                 printf("Warning: lsc_interrupt needs to be off when "
3317                                 " using tx_first. Disabling.\n");
3318                 lsc_interrupt = 0;
3319         }
3320
3321         if (!nb_rxq && !nb_txq)
3322                 printf("Warning: Either rx or tx queues should be non-zero\n");
3323
3324         if (nb_rxq > 1 && nb_rxq > nb_txq)
3325                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3326                        "but nb_txq=%d will prevent to fully test it.\n",
3327                        nb_rxq, nb_txq);
3328
3329         init_config();
3330
3331         if (hot_plug) {
3332                 ret = rte_dev_hotplug_handle_enable();
3333                 if (ret) {
3334                         RTE_LOG(ERR, EAL,
3335                                 "fail to enable hotplug handling.");
3336                         return -1;
3337                 }
3338
3339                 ret = rte_dev_event_monitor_start();
3340                 if (ret) {
3341                         RTE_LOG(ERR, EAL,
3342                                 "fail to start device event monitoring.");
3343                         return -1;
3344                 }
3345
3346                 ret = rte_dev_event_callback_register(NULL,
3347                         dev_event_callback, NULL);
3348                 if (ret) {
3349                         RTE_LOG(ERR, EAL,
3350                                 "fail  to register device event callback\n");
3351                         return -1;
3352                 }
3353         }
3354
3355         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3356                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3357
3358         /* set all ports to promiscuous mode by default */
3359         RTE_ETH_FOREACH_DEV(port_id) {
3360                 ret = rte_eth_promiscuous_enable(port_id);
3361                 if (ret != 0)
3362                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3363                                 port_id, rte_strerror(-ret));
3364         }
3365
3366         /* Init metrics library */
3367         rte_metrics_init(rte_socket_id());
3368
3369 #ifdef RTE_LIBRTE_LATENCY_STATS
3370         if (latencystats_enabled != 0) {
3371                 int ret = rte_latencystats_init(1, NULL);
3372                 if (ret)
3373                         printf("Warning: latencystats init()"
3374                                 " returned error %d\n", ret);
3375                 printf("Latencystats running on lcore %d\n",
3376                         latencystats_lcore_id);
3377         }
3378 #endif
3379
3380         /* Setup bitrate stats */
3381 #ifdef RTE_LIBRTE_BITRATE
3382         if (bitrate_enabled != 0) {
3383                 bitrate_data = rte_stats_bitrate_create();
3384                 if (bitrate_data == NULL)
3385                         rte_exit(EXIT_FAILURE,
3386                                 "Could not allocate bitrate data.\n");
3387                 rte_stats_bitrate_reg(bitrate_data);
3388         }
3389 #endif
3390
3391 #ifdef RTE_LIBRTE_CMDLINE
3392         if (strlen(cmdline_filename) != 0)
3393                 cmdline_read_from_file(cmdline_filename);
3394
3395         if (interactive == 1) {
3396                 if (auto_start) {
3397                         printf("Start automatic packet forwarding\n");
3398                         start_packet_forwarding(0);
3399                 }
3400                 prompt();
3401                 pmd_test_exit();
3402         } else
3403 #endif
3404         {
3405                 char c;
3406                 int rc;
3407
3408                 f_quit = 0;
3409
3410                 printf("No commandline core given, start packet forwarding\n");
3411                 start_packet_forwarding(tx_first);
3412                 if (stats_period != 0) {
3413                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3414                         uint64_t timer_period;
3415
3416                         /* Convert to number of cycles */
3417                         timer_period = stats_period * rte_get_timer_hz();
3418
3419                         while (f_quit == 0) {
3420                                 cur_time = rte_get_timer_cycles();
3421                                 diff_time += cur_time - prev_time;
3422
3423                                 if (diff_time >= timer_period) {
3424                                         print_stats();
3425                                         /* Reset the timer */
3426                                         diff_time = 0;
3427                                 }
3428                                 /* Sleep to avoid unnecessary checks */
3429                                 prev_time = cur_time;
3430                                 sleep(1);
3431                         }
3432                 }
3433
3434                 printf("Press enter to exit\n");
3435                 rc = read(0, &c, 1);
3436                 pmd_test_exit();
3437                 if (rc < 0)
3438                         return 1;
3439         }
3440
3441         return 0;
3442 }