app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_malloc_heap.h>
  31 #include <rte_memory.h>
  32 #include <rte_memcpy.h>
  33 #include <rte_launch.h>
  34 #include <rte_eal.h>
  35 #include <rte_alarm.h>
  36 #include <rte_per_lcore.h>
  37 #include <rte_lcore.h>
  38 #include <rte_atomic.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_LIBRTE_IXGBE_PMD
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIBRTE_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #include <rte_metrics.h>
  58 #ifdef RTE_LIBRTE_BITRATE
  59 #include <rte_bitrate.h>
  60 #endif
  61 #ifdef RTE_LIBRTE_LATENCY_STATS
  62 #include <rte_latencystats.h>
  63 #endif
  64
  65 #include "testpmd.h"
  66
  67 #ifndef MAP_HUGETLB
  68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  69 #define HUGE_FLAG (0x40000)
  70 #else
  71 #define HUGE_FLAG MAP_HUGETLB
  72 #endif
  73
  74 #ifndef MAP_HUGE_SHIFT
  75 /* older kernels (or FreeBSD) will not have this define */
  76 #define HUGE_SHIFT (26)
  77 #else
  78 #define HUGE_SHIFT MAP_HUGE_SHIFT
  79 #endif
  80
  81 #define EXTMEM_HEAP_NAME "extmem"
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192
 193 struct fwd_config cur_fwd_config;
 194 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 195 uint32_t retry_enabled;
 196 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 197 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 198
 199 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 200 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 201                                       * specified on command-line. */
 202 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 203
 204 /*
 205  * In container, it cannot terminate the process which running with 'stats-period'
 206  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 207  */
 208 uint8_t f_quit;
 209
 210 /*
 211  * Configuration of packet segments used by the "txonly" processing engine.
 212  */
 213 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 214 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 215         TXONLY_DEF_PACKET_LEN,
 216 };
 217 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 218
 219 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 220 /**< Split policy for packets to TX. */
 221
 222 uint8_t txonly_multi_flow;
 223 /**< Whether multiple flows are generated in TXONLY mode. */
 224
 225 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 226 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 227
 228 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 229 uint8_t dcb_config = 0;
 230
 231 /* Whether the dcb is in testing status */
 232 uint8_t dcb_test = 0;
 233
 234 /*
 235  * Configurable number of RX/TX queues.
 236  */
 237 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 238 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 239
 240 /*
 241  * Configurable number of RX/TX ring descriptors.
 242  * Defaults are supplied by drivers via ethdev.
 243  */
 244 #define RTE_TEST_RX_DESC_DEFAULT 0
 245 #define RTE_TEST_TX_DESC_DEFAULT 0
 246 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 247 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 248
 249 #define RTE_PMD_PARAM_UNSET -1
 250 /*
 251  * Configurable values of RX and TX ring threshold registers.
 252  */
 253
 254 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 255 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 256 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 257
 258 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 259 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 260 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 261
 262 /*
 263  * Configurable value of RX free threshold.
 264  */
 265 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 266
 267 /*
 268  * Configurable value of RX drop enable.
 269  */
 270 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 271
 272 /*
 273  * Configurable value of TX free threshold.
 274  */
 275 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 276
 277 /*
 278  * Configurable value of TX RS bit threshold.
 279  */
 280 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 281
 282 /*
 283  * Configurable value of buffered packets before sending.
 284  */
 285 uint16_t noisy_tx_sw_bufsz;
 286
 287 /*
 288  * Configurable value of packet buffer timeout.
 289  */
 290 uint16_t noisy_tx_sw_buf_flush_time;
 291
 292 /*
 293  * Configurable value for size of VNF internal memory area
 294  * used for simulating noisy neighbour behaviour
 295  */
 296 uint64_t noisy_lkup_mem_sz;
 297
 298 /*
 299  * Configurable value of number of random writes done in
 300  * VNF simulation memory area.
 301  */
 302 uint64_t noisy_lkup_num_writes;
 303
 304 /*
 305  * Configurable value of number of random reads done in
 306  * VNF simulation memory area.
 307  */
 308 uint64_t noisy_lkup_num_reads;
 309
 310 /*
 311  * Configurable value of number of random reads/writes done in
 312  * VNF simulation memory area.
 313  */
 314 uint64_t noisy_lkup_num_reads_writes;
 315
 316 /*
 317  * Receive Side Scaling (RSS) configuration.
 318  */
 319 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 320
 321 /*
 322  * Port topology configuration
 323  */
 324 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 325
 326 /*
 327  * Avoids to flush all the RX streams before starts forwarding.
 328  */
 329 uint8_t no_flush_rx = 0; /* flush by default */
 330
 331 /*
 332  * Flow API isolated mode.
 333  */
 334 uint8_t flow_isolate_all;
 335
 336 /*
 337  * Avoids to check link status when starting/stopping a port.
 338  */
 339 uint8_t no_link_check = 0; /* check by default */
 340
 341 /*
 342  * Enable link status change notification
 343  */
 344 uint8_t lsc_interrupt = 1; /* enabled by default */
 345
 346 /*
 347  * Enable device removal notification.
 348  */
 349 uint8_t rmv_interrupt = 1; /* enabled by default */
 350
 351 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 352
 353 /* After attach, port setup is called on event or by iterator */
 354 bool setup_on_probe_event = true;
 355
 356 /* Pretty printing of ethdev events */
 357 static const char * const eth_event_desc[] = {
 358         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 359         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 360         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 361         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 362         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 363         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 364         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 365         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 366         [RTE_ETH_EVENT_NEW] = "device probed",
 367         [RTE_ETH_EVENT_DESTROY] = "device released",
 368         [RTE_ETH_EVENT_MAX] = NULL,
 369 };
 370
 371 /*
 372  * Display or mask ether events
 373  * Default to all events except VF_MBOX
 374  */
 375 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 376                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 377                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 378                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 379                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 380                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 381                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 382 /*
 383  * Decide if all memory are locked for performance.
 384  */
 385 int do_mlockall = 0;
 386
 387 /*
 388  * NIC bypass mode configuration options.
 389  */
 390
 391 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 392 /* The NIC bypass watchdog timeout. */
 393 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 394 #endif
 395
 396
 397 #ifdef RTE_LIBRTE_LATENCY_STATS
 398
 399 /*
 400  * Set when latency stats is enabled in the commandline
 401  */
 402 uint8_t latencystats_enabled;
 403
 404 /*
 405  * Lcore ID to serive latency statistics.
 406  */
 407 lcoreid_t latencystats_lcore_id = -1;
 408
 409 #endif
 410
 411 /*
 412  * Ethernet device configuration.
 413  */
 414 struct rte_eth_rxmode rx_mode = {
 415         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 416 };
 417
 418 struct rte_eth_txmode tx_mode = {
 419         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 420 };
 421
 422 struct rte_fdir_conf fdir_conf = {
 423         .mode = RTE_FDIR_MODE_NONE,
 424         .pballoc = RTE_FDIR_PBALLOC_64K,
 425         .status = RTE_FDIR_REPORT_STATUS,
 426         .mask = {
 427                 .vlan_tci_mask = 0xFFEF,
 428                 .ipv4_mask     = {
 429                         .src_ip = 0xFFFFFFFF,
 430                         .dst_ip = 0xFFFFFFFF,
 431                 },
 432                 .ipv6_mask     = {
 433                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 434                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 435                 },
 436                 .src_port_mask = 0xFFFF,
 437                 .dst_port_mask = 0xFFFF,
 438                 .mac_addr_byte_mask = 0xFF,
 439                 .tunnel_type_mask = 1,
 440                 .tunnel_id_mask = 0xFFFFFFFF,
 441         },
 442         .drop_queue = 127,
 443 };
 444
 445 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 446
 447 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 448 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 449
 450 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 451 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 452
 453 uint16_t nb_tx_queue_stats_mappings = 0;
 454 uint16_t nb_rx_queue_stats_mappings = 0;
 455
 456 /*
 457  * Display zero values by default for xstats
 458  */
 459 uint8_t xstats_hide_zero;
 460
 461 unsigned int num_sockets = 0;
 462 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 463
 464 #ifdef RTE_LIBRTE_BITRATE
 465 /* Bitrate statistics */
 466 struct rte_stats_bitrates *bitrate_data;
 467 lcoreid_t bitrate_lcore_id;
 468 uint8_t bitrate_enabled;
 469 #endif
 470
 471 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 472 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 473
 474 struct vxlan_encap_conf vxlan_encap_conf = {
 475         .select_ipv4 = 1,
 476         .select_vlan = 0,
 477         .select_tos_ttl = 0,
 478         .vni = "\x00\x00\x00",
 479         .udp_src = 0,
 480         .udp_dst = RTE_BE16(4789),
 481         .ipv4_src = IPv4(127, 0, 0, 1),
 482         .ipv4_dst = IPv4(255, 255, 255, 255),
 483         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 484                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 485         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 486                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 487         .vlan_tci = 0,
 488         .ip_tos = 0,
 489         .ip_ttl = 255,
 490         .eth_src = "\x00\x00\x00\x00\x00\x00",
 491         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 492 };
 493
 494 struct nvgre_encap_conf nvgre_encap_conf = {
 495         .select_ipv4 = 1,
 496         .select_vlan = 0,
 497         .tni = "\x00\x00\x00",
 498         .ipv4_src = IPv4(127, 0, 0, 1),
 499         .ipv4_dst = IPv4(255, 255, 255, 255),
 500         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 501                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 502         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 503                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 504         .vlan_tci = 0,
 505         .eth_src = "\x00\x00\x00\x00\x00\x00",
 506         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 507 };
 508
 509 /* Forward function declarations */
 510 static void setup_attached_port(portid_t pi);
 511 static void map_port_queue_stats_mapping_registers(portid_t pi,
 512                                                    struct rte_port *port);
 513 static void check_all_ports_link_status(uint32_t port_mask);
 514 static int eth_event_callback(portid_t port_id,
 515                               enum rte_eth_event_type type,
 516                               void *param, void *ret_param);
 517 static void dev_event_callback(const char *device_name,
 518                                 enum rte_dev_event_type type,
 519                                 void *param);
 520
 521 /*
 522  * Check if all the ports are started.
 523  * If yes, return positive value. If not, return zero.
 524  */
 525 static int all_ports_started(void);
 526
 527 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 528 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN;
 529
 530 /*
 531  * Helper function to check if socket is already discovered.
 532  * If yes, return positive value. If not, return zero.
 533  */
 534 int
 535 new_socket_id(unsigned int socket_id)
 536 {
 537         unsigned int i;
 538
 539         for (i = 0; i < num_sockets; i++) {
 540                 if (socket_ids[i] == socket_id)
 541                         return 0;
 542         }
 543         return 1;
 544 }
 545
 546 /*
 547  * Setup default configuration.
 548  */
 549 static void
 550 set_default_fwd_lcores_config(void)
 551 {
 552         unsigned int i;
 553         unsigned int nb_lc;
 554         unsigned int sock_num;
 555
 556         nb_lc = 0;
 557         for (i = 0; i < RTE_MAX_LCORE; i++) {
 558                 if (!rte_lcore_is_enabled(i))
 559                         continue;
 560                 sock_num = rte_lcore_to_socket_id(i);
 561                 if (new_socket_id(sock_num)) {
 562                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 563                                 rte_exit(EXIT_FAILURE,
 564                                          "Total sockets greater than %u\n",
 565                                          RTE_MAX_NUMA_NODES);
 566                         }
 567                         socket_ids[num_sockets++] = sock_num;
 568                 }
 569                 if (i == rte_get_master_lcore())
 570                         continue;
 571                 fwd_lcores_cpuids[nb_lc++] = i;
 572         }
 573         nb_lcores = (lcoreid_t) nb_lc;
 574         nb_cfg_lcores = nb_lcores;
 575         nb_fwd_lcores = 1;
 576 }
 577
 578 static void
 579 set_def_peer_eth_addrs(void)
 580 {
 581         portid_t i;
 582
 583         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 584                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 585                 peer_eth_addrs[i].addr_bytes[5] = i;
 586         }
 587 }
 588
 589 static void
 590 set_default_fwd_ports_config(void)
 591 {
 592         portid_t pt_id;
 593         int i = 0;
 594
 595         RTE_ETH_FOREACH_DEV(pt_id) {
 596                 fwd_ports_ids[i++] = pt_id;
 597
 598                 /* Update sockets info according to the attached device */
 599                 int socket_id = rte_eth_dev_socket_id(pt_id);
 600                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 601                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 602                                 rte_exit(EXIT_FAILURE,
 603                                          "Total sockets greater than %u\n",
 604                                          RTE_MAX_NUMA_NODES);
 605                         }
 606                         socket_ids[num_sockets++] = socket_id;
 607                 }
 608         }
 609
 610         nb_cfg_ports = nb_ports;
 611         nb_fwd_ports = nb_ports;
 612 }
 613
 614 void
 615 set_def_fwd_config(void)
 616 {
 617         set_default_fwd_lcores_config();
 618         set_def_peer_eth_addrs();
 619         set_default_fwd_ports_config();
 620 }
 621
 622 /* extremely pessimistic estimation of memory required to create a mempool */
 623 static int
 624 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 625 {
 626         unsigned int n_pages, mbuf_per_pg, leftover;
 627         uint64_t total_mem, mbuf_mem, obj_sz;
 628
 629         /* there is no good way to predict how much space the mempool will
 630          * occupy because it will allocate chunks on the fly, and some of those
 631          * will come from default DPDK memory while some will come from our
 632          * external memory, so just assume 128MB will be enough for everyone.
 633          */
 634         uint64_t hdr_mem = 128 << 20;
 635
 636         /* account for possible non-contiguousness */
 637         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 638         if (obj_sz > pgsz) {
 639                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 640                 return -1;
 641         }
 642
 643         mbuf_per_pg = pgsz / obj_sz;
 644         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 645         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 646
 647         mbuf_mem = n_pages * pgsz;
 648
 649         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 650
 651         if (total_mem > SIZE_MAX) {
 652                 TESTPMD_LOG(ERR, "Memory size too big\n");
 653                 return -1;
 654         }
 655         *out = (size_t)total_mem;
 656
 657         return 0;
 658 }
 659
 660 static int
 661 pagesz_flags(uint64_t page_sz)
 662 {
 663         /* as per mmap() manpage, all page sizes are log2 of page size
 664          * shifted by MAP_HUGE_SHIFT
 665          */
 666         int log2 = rte_log2_u64(page_sz);
 667
 668         return (log2 << HUGE_SHIFT);
 669 }
 670
 671 static void *
 672 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 673 {
 674         void *addr;
 675         int flags;
 676
 677         /* allocate anonymous hugepages */
 678         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 679         if (huge)
 680                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 681
 682         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 683         if (addr == MAP_FAILED)
 684                 return NULL;
 685
 686         return addr;
 687 }
 688
 689 struct extmem_param {
 690         void *addr;
 691         size_t len;
 692         size_t pgsz;
 693         rte_iova_t *iova_table;
 694         unsigned int iova_table_len;
 695 };
 696
 697 static int
 698 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 699                 bool huge)
 700 {
 701         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 702                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 703         unsigned int cur_page, n_pages, pgsz_idx;
 704         size_t mem_sz, cur_pgsz;
 705         rte_iova_t *iovas = NULL;
 706         void *addr;
 707         int ret;
 708
 709         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 710                 /* skip anything that is too big */
 711                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 712                         continue;
 713
 714                 cur_pgsz = pgsizes[pgsz_idx];
 715
 716                 /* if we were told not to allocate hugepages, override */
 717                 if (!huge)
 718                         cur_pgsz = sysconf(_SC_PAGESIZE);
 719
 720                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 721                 if (ret < 0) {
 722                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 723                         return -1;
 724                 }
 725
 726                 /* allocate our memory */
 727                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 728
 729                 /* if we couldn't allocate memory with a specified page size,
 730                  * that doesn't mean we can't do it with other page sizes, so
 731                  * try another one.
 732                  */
 733                 if (addr == NULL)
 734                         continue;
 735
 736                 /* store IOVA addresses for every page in this memory area */
 737                 n_pages = mem_sz / cur_pgsz;
 738
 739                 iovas = malloc(sizeof(*iovas) * n_pages);
 740
 741                 if (iovas == NULL) {
 742                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 743                         goto fail;
 744                 }
 745                 /* lock memory if it's not huge pages */
 746                 if (!huge)
 747                         mlock(addr, mem_sz);
 748
 749                 /* populate IOVA addresses */
 750                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 751                         rte_iova_t iova;
 752                         size_t offset;
 753                         void *cur;
 754
 755                         offset = cur_pgsz * cur_page;
 756                         cur = RTE_PTR_ADD(addr, offset);
 757
 758                         /* touch the page before getting its IOVA */
 759                         *(volatile char *)cur = 0;
 760
 761                         iova = rte_mem_virt2iova(cur);
 762
 763                         iovas[cur_page] = iova;
 764                 }
 765
 766                 break;
 767         }
 768         /* if we couldn't allocate anything */
 769         if (iovas == NULL)
 770                 return -1;
 771
 772         param->addr = addr;
 773         param->len = mem_sz;
 774         param->pgsz = cur_pgsz;
 775         param->iova_table = iovas;
 776         param->iova_table_len = n_pages;
 777
 778         return 0;
 779 fail:
 780         if (iovas)
 781                 free(iovas);
 782         if (addr)
 783                 munmap(addr, mem_sz);
 784
 785         return -1;
 786 }
 787
 788 static int
 789 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 790 {
 791         struct extmem_param param;
 792         int socket_id, ret;
 793
 794         memset(&param, 0, sizeof(param));
 795
 796         /* check if our heap exists */
 797         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 798         if (socket_id < 0) {
 799                 /* create our heap */
 800                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 801                 if (ret < 0) {
 802                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 803                         return -1;
 804                 }
 805         }
 806
 807         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 808         if (ret < 0) {
 809                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 810                 return -1;
 811         }
 812
 813         /* we now have a valid memory area, so add it to heap */
 814         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 815                         param.addr, param.len, param.iova_table,
 816                         param.iova_table_len, param.pgsz);
 817
 818         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 819
 820         /* not needed any more */
 821         free(param.iova_table);
 822
 823         if (ret < 0) {
 824                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 825                 munmap(param.addr, param.len);
 826                 return -1;
 827         }
 828
 829         /* success */
 830
 831         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 832                         param.len >> 20);
 833
 834         return 0;
 835 }
 836
 837 /*
 838  * Configuration initialisation done once at init time.
 839  */
 840 static struct rte_mempool *
 841 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 842                  unsigned int socket_id)
 843 {
 844         char pool_name[RTE_MEMPOOL_NAMESIZE];
 845         struct rte_mempool *rte_mp = NULL;
 846         uint32_t mb_size;
 847
 848         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 849         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 850
 851         TESTPMD_LOG(INFO,
 852                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 853                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 854
 855         switch (mp_alloc_type) {
 856         case MP_ALLOC_NATIVE:
 857                 {
 858                         /* wrapper to rte_mempool_create() */
 859                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 860                                         rte_mbuf_best_mempool_ops());
 861                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 862                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 863                         break;
 864                 }
 865         case MP_ALLOC_ANON:
 866                 {
 867                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 868                                 mb_size, (unsigned int) mb_mempool_cache,
 869                                 sizeof(struct rte_pktmbuf_pool_private),
 870                                 socket_id, 0);
 871                         if (rte_mp == NULL)
 872                                 goto err;
 873
 874                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 875                                 rte_mempool_free(rte_mp);
 876                                 rte_mp = NULL;
 877                                 goto err;
 878                         }
 879                         rte_pktmbuf_pool_init(rte_mp, NULL);
 880                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 881                         break;
 882                 }
 883         case MP_ALLOC_XMEM:
 884         case MP_ALLOC_XMEM_HUGE:
 885                 {
 886                         int heap_socket;
 887                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 888
 889                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 890                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 891
 892                         heap_socket =
 893                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 894                         if (heap_socket < 0)
 895                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 896
 897                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 898                                         rte_mbuf_best_mempool_ops());
 899                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 900                                         mb_mempool_cache, 0, mbuf_seg_size,
 901                                         heap_socket);
 902                         break;
 903                 }
 904         default:
 905                 {
 906                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 907                 }
 908         }
 909
 910 err:
 911         if (rte_mp == NULL) {
 912                 rte_exit(EXIT_FAILURE,
 913                         "Creation of mbuf pool for socket %u failed: %s\n",
 914                         socket_id, rte_strerror(rte_errno));
 915         } else if (verbose_level > 0) {
 916                 rte_mempool_dump(stdout, rte_mp);
 917         }
 918         return rte_mp;
 919 }
 920
 921 /*
 922  * Check given socket id is valid or not with NUMA mode,
 923  * if valid, return 0, else return -1
 924  */
 925 static int
 926 check_socket_id(const unsigned int socket_id)
 927 {
 928         static int warning_once = 0;
 929
 930         if (new_socket_id(socket_id)) {
 931                 if (!warning_once && numa_support)
 932                         printf("Warning: NUMA should be configured manually by"
 933                                " using --port-numa-config and"
 934                                " --ring-numa-config parameters along with"
 935                                " --numa.\n");
 936                 warning_once = 1;
 937                 return -1;
 938         }
 939         return 0;
 940 }
 941
 942 /*
 943  * Get the allowed maximum number of RX queues.
 944  * *pid return the port id which has minimal value of
 945  * max_rx_queues in all ports.
 946  */
 947 queueid_t
 948 get_allowed_max_nb_rxq(portid_t *pid)
 949 {
 950         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
 951         portid_t pi;
 952         struct rte_eth_dev_info dev_info;
 953
 954         RTE_ETH_FOREACH_DEV(pi) {
 955                 rte_eth_dev_info_get(pi, &dev_info);
 956                 if (dev_info.max_rx_queues < allowed_max_rxq) {
 957                         allowed_max_rxq = dev_info.max_rx_queues;
 958                         *pid = pi;
 959                 }
 960         }
 961         return allowed_max_rxq;
 962 }
 963
 964 /*
 965  * Check input rxq is valid or not.
 966  * If input rxq is not greater than any of maximum number
 967  * of RX queues of all ports, it is valid.
 968  * if valid, return 0, else return -1
 969  */
 970 int
 971 check_nb_rxq(queueid_t rxq)
 972 {
 973         queueid_t allowed_max_rxq;
 974         portid_t pid = 0;
 975
 976         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
 977         if (rxq > allowed_max_rxq) {
 978                 printf("Fail: input rxq (%u) can't be greater "
 979                        "than max_rx_queues (%u) of port %u\n",
 980                        rxq,
 981                        allowed_max_rxq,
 982                        pid);
 983                 return -1;
 984         }
 985         return 0;
 986 }
 987
 988 /*
 989  * Get the allowed maximum number of TX queues.
 990  * *pid return the port id which has minimal value of
 991  * max_tx_queues in all ports.
 992  */
 993 queueid_t
 994 get_allowed_max_nb_txq(portid_t *pid)
 995 {
 996         queueid_t allowed_max_txq = MAX_QUEUE_ID;
 997         portid_t pi;
 998         struct rte_eth_dev_info dev_info;
 999
1000         RTE_ETH_FOREACH_DEV(pi) {
1001                 rte_eth_dev_info_get(pi, &dev_info);
1002                 if (dev_info.max_tx_queues < allowed_max_txq) {
1003                         allowed_max_txq = dev_info.max_tx_queues;
1004                         *pid = pi;
1005                 }
1006         }
1007         return allowed_max_txq;
1008 }
1009
1010 /*
1011  * Check input txq is valid or not.
1012  * If input txq is not greater than any of maximum number
1013  * of TX queues of all ports, it is valid.
1014  * if valid, return 0, else return -1
1015  */
1016 int
1017 check_nb_txq(queueid_t txq)
1018 {
1019         queueid_t allowed_max_txq;
1020         portid_t pid = 0;
1021
1022         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1023         if (txq > allowed_max_txq) {
1024                 printf("Fail: input txq (%u) can't be greater "
1025                        "than max_tx_queues (%u) of port %u\n",
1026                        txq,
1027                        allowed_max_txq,
1028                        pid);
1029                 return -1;
1030         }
1031         return 0;
1032 }
1033
1034 static void
1035 init_config(void)
1036 {
1037         portid_t pid;
1038         struct rte_port *port;
1039         struct rte_mempool *mbp;
1040         unsigned int nb_mbuf_per_pool;
1041         lcoreid_t  lc_id;
1042         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1043         struct rte_gro_param gro_param;
1044         uint32_t gso_types;
1045         int k;
1046
1047         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1048
1049         /* Configuration of logical cores. */
1050         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1051                                 sizeof(struct fwd_lcore *) * nb_lcores,
1052                                 RTE_CACHE_LINE_SIZE);
1053         if (fwd_lcores == NULL) {
1054                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1055                                                         "failed\n", nb_lcores);
1056         }
1057         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1058                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1059                                                sizeof(struct fwd_lcore),
1060                                                RTE_CACHE_LINE_SIZE);
1061                 if (fwd_lcores[lc_id] == NULL) {
1062                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1063                                                                 "failed\n");
1064                 }
1065                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1066         }
1067
1068         RTE_ETH_FOREACH_DEV(pid) {
1069                 port = &ports[pid];
1070                 /* Apply default TxRx configuration for all ports */
1071                 port->dev_conf.txmode = tx_mode;
1072                 port->dev_conf.rxmode = rx_mode;
1073                 rte_eth_dev_info_get(pid, &port->dev_info);
1074
1075                 if (!(port->dev_info.tx_offload_capa &
1076                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1077                         port->dev_conf.txmode.offloads &=
1078                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1079                 if (!(port->dev_info.tx_offload_capa &
1080                         DEV_TX_OFFLOAD_MATCH_METADATA))
1081                         port->dev_conf.txmode.offloads &=
1082                                 ~DEV_TX_OFFLOAD_MATCH_METADATA;
1083                 if (numa_support) {
1084                         if (port_numa[pid] != NUMA_NO_CONFIG)
1085                                 port_per_socket[port_numa[pid]]++;
1086                         else {
1087                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1088
1089                                 /*
1090                                  * if socket_id is invalid,
1091                                  * set to the first available socket.
1092                                  */
1093                                 if (check_socket_id(socket_id) < 0)
1094                                         socket_id = socket_ids[0];
1095                                 port_per_socket[socket_id]++;
1096                         }
1097                 }
1098
1099                 /* Apply Rx offloads configuration */
1100                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1101                         port->rx_conf[k].offloads =
1102                                 port->dev_conf.rxmode.offloads;
1103                 /* Apply Tx offloads configuration */
1104                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1105                         port->tx_conf[k].offloads =
1106                                 port->dev_conf.txmode.offloads;
1107
1108                 /* set flag to initialize port/queue */
1109                 port->need_reconfig = 1;
1110                 port->need_reconfig_queues = 1;
1111                 port->tx_metadata = 0;
1112         }
1113
1114         /*
1115          * Create pools of mbuf.
1116          * If NUMA support is disabled, create a single pool of mbuf in
1117          * socket 0 memory by default.
1118          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1119          *
1120          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1121          * nb_txd can be configured at run time.
1122          */
1123         if (param_total_num_mbufs)
1124                 nb_mbuf_per_pool = param_total_num_mbufs;
1125         else {
1126                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1127                         (nb_lcores * mb_mempool_cache) +
1128                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1129                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1130         }
1131
1132         if (numa_support) {
1133                 uint8_t i;
1134
1135                 for (i = 0; i < num_sockets; i++)
1136                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1137                                                        nb_mbuf_per_pool,
1138                                                        socket_ids[i]);
1139         } else {
1140                 if (socket_num == UMA_NO_CONFIG)
1141                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1142                                                        nb_mbuf_per_pool, 0);
1143                 else
1144                         mempools[socket_num] = mbuf_pool_create
1145                                                         (mbuf_data_size,
1146                                                          nb_mbuf_per_pool,
1147                                                          socket_num);
1148         }
1149
1150         init_port_config();
1151
1152         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1153                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1154         /*
1155          * Records which Mbuf pool to use by each logical core, if needed.
1156          */
1157         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1158                 mbp = mbuf_pool_find(
1159                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1160
1161                 if (mbp == NULL)
1162                         mbp = mbuf_pool_find(0);
1163                 fwd_lcores[lc_id]->mbp = mbp;
1164                 /* initialize GSO context */
1165                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1166                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1167                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1168                 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN -
1169                         ETHER_CRC_LEN;
1170                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1171         }
1172
1173         /* Configuration of packet forwarding streams. */
1174         if (init_fwd_streams() < 0)
1175                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1176
1177         fwd_config_setup();
1178
1179         /* create a gro context for each lcore */
1180         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1181         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1182         gro_param.max_item_per_flow = MAX_PKT_BURST;
1183         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1184                 gro_param.socket_id = rte_lcore_to_socket_id(
1185                                 fwd_lcores_cpuids[lc_id]);
1186                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1187                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1188                         rte_exit(EXIT_FAILURE,
1189                                         "rte_gro_ctx_create() failed\n");
1190                 }
1191         }
1192
1193 #if defined RTE_LIBRTE_PMD_SOFTNIC
1194         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1195                 RTE_ETH_FOREACH_DEV(pid) {
1196                         port = &ports[pid];
1197                         const char *driver = port->dev_info.driver_name;
1198
1199                         if (strcmp(driver, "net_softnic") == 0)
1200                                 port->softport.fwd_lcore_arg = fwd_lcores;
1201                 }
1202         }
1203 #endif
1204
1205 }
1206
1207
1208 void
1209 reconfig(portid_t new_port_id, unsigned socket_id)
1210 {
1211         struct rte_port *port;
1212
1213         /* Reconfiguration of Ethernet ports. */
1214         port = &ports[new_port_id];
1215         rte_eth_dev_info_get(new_port_id, &port->dev_info);
1216
1217         /* set flag to initialize port/queue */
1218         port->need_reconfig = 1;
1219         port->need_reconfig_queues = 1;
1220         port->socket_id = socket_id;
1221
1222         init_port_config();
1223 }
1224
1225
1226 int
1227 init_fwd_streams(void)
1228 {
1229         portid_t pid;
1230         struct rte_port *port;
1231         streamid_t sm_id, nb_fwd_streams_new;
1232         queueid_t q;
1233
1234         /* set socket id according to numa or not */
1235         RTE_ETH_FOREACH_DEV(pid) {
1236                 port = &ports[pid];
1237                 if (nb_rxq > port->dev_info.max_rx_queues) {
1238                         printf("Fail: nb_rxq(%d) is greater than "
1239                                 "max_rx_queues(%d)\n", nb_rxq,
1240                                 port->dev_info.max_rx_queues);
1241                         return -1;
1242                 }
1243                 if (nb_txq > port->dev_info.max_tx_queues) {
1244                         printf("Fail: nb_txq(%d) is greater than "
1245                                 "max_tx_queues(%d)\n", nb_txq,
1246                                 port->dev_info.max_tx_queues);
1247                         return -1;
1248                 }
1249                 if (numa_support) {
1250                         if (port_numa[pid] != NUMA_NO_CONFIG)
1251                                 port->socket_id = port_numa[pid];
1252                         else {
1253                                 port->socket_id = rte_eth_dev_socket_id(pid);
1254
1255                                 /*
1256                                  * if socket_id is invalid,
1257                                  * set to the first available socket.
1258                                  */
1259                                 if (check_socket_id(port->socket_id) < 0)
1260                                         port->socket_id = socket_ids[0];
1261                         }
1262                 }
1263                 else {
1264                         if (socket_num == UMA_NO_CONFIG)
1265                                 port->socket_id = 0;
1266                         else
1267                                 port->socket_id = socket_num;
1268                 }
1269         }
1270
1271         q = RTE_MAX(nb_rxq, nb_txq);
1272         if (q == 0) {
1273                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1274                 return -1;
1275         }
1276         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1277         if (nb_fwd_streams_new == nb_fwd_streams)
1278                 return 0;
1279         /* clear the old */
1280         if (fwd_streams != NULL) {
1281                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1282                         if (fwd_streams[sm_id] == NULL)
1283                                 continue;
1284                         rte_free(fwd_streams[sm_id]);
1285                         fwd_streams[sm_id] = NULL;
1286                 }
1287                 rte_free(fwd_streams);
1288                 fwd_streams = NULL;
1289         }
1290
1291         /* init new */
1292         nb_fwd_streams = nb_fwd_streams_new;
1293         if (nb_fwd_streams) {
1294                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1295                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1296                         RTE_CACHE_LINE_SIZE);
1297                 if (fwd_streams == NULL)
1298                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1299                                  " (struct fwd_stream *)) failed\n",
1300                                  nb_fwd_streams);
1301
1302                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1303                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1304                                 " struct fwd_stream", sizeof(struct fwd_stream),
1305                                 RTE_CACHE_LINE_SIZE);
1306                         if (fwd_streams[sm_id] == NULL)
1307                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1308                                          "(struct fwd_stream) failed\n");
1309                 }
1310         }
1311
1312         return 0;
1313 }
1314
1315 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1316 static void
1317 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1318 {
1319         unsigned int total_burst;
1320         unsigned int nb_burst;
1321         unsigned int burst_stats[3];
1322         uint16_t pktnb_stats[3];
1323         uint16_t nb_pkt;
1324         int burst_percent[3];
1325
1326         /*
1327          * First compute the total number of packet bursts and the
1328          * two highest numbers of bursts of the same number of packets.
1329          */
1330         total_burst = 0;
1331         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1332         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1333         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1334                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1335                 if (nb_burst == 0)
1336                         continue;
1337                 total_burst += nb_burst;
1338                 if (nb_burst > burst_stats[0]) {
1339                         burst_stats[1] = burst_stats[0];
1340                         pktnb_stats[1] = pktnb_stats[0];
1341                         burst_stats[0] = nb_burst;
1342                         pktnb_stats[0] = nb_pkt;
1343                 } else if (nb_burst > burst_stats[1]) {
1344                         burst_stats[1] = nb_burst;
1345                         pktnb_stats[1] = nb_pkt;
1346                 }
1347         }
1348         if (total_burst == 0)
1349                 return;
1350         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1351         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1352                burst_percent[0], (int) pktnb_stats[0]);
1353         if (burst_stats[0] == total_burst) {
1354                 printf("]\n");
1355                 return;
1356         }
1357         if (burst_stats[0] + burst_stats[1] == total_burst) {
1358                 printf(" + %d%% of %d pkts]\n",
1359                        100 - burst_percent[0], pktnb_stats[1]);
1360                 return;
1361         }
1362         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1363         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1364         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1365                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1366                 return;
1367         }
1368         printf(" + %d%% of %d pkts + %d%% of others]\n",
1369                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1370 }
1371 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1372
1373 static void
1374 fwd_stream_stats_display(streamid_t stream_id)
1375 {
1376         struct fwd_stream *fs;
1377         static const char *fwd_top_stats_border = "-------";
1378
1379         fs = fwd_streams[stream_id];
1380         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1381             (fs->fwd_dropped == 0))
1382                 return;
1383         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1384                "TX Port=%2d/Queue=%2d %s\n",
1385                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1386                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1387         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1388                " TX-dropped: %-14"PRIu64,
1389                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1390
1391         /* if checksum mode */
1392         if (cur_fwd_eng == &csum_fwd_engine) {
1393                 printf("  RX- bad IP checksum: %-14"PRIu64
1394                        "  Rx- bad L4 checksum: %-14"PRIu64
1395                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1396                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1397                         fs->rx_bad_outer_l4_csum);
1398         } else {
1399                 printf("\n");
1400         }
1401
1402 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1403         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1404         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1405 #endif
1406 }
1407
1408 void
1409 fwd_stats_display(void)
1410 {
1411         static const char *fwd_stats_border = "----------------------";
1412         static const char *acc_stats_border = "+++++++++++++++";
1413         struct {
1414                 struct fwd_stream *rx_stream;
1415                 struct fwd_stream *tx_stream;
1416                 uint64_t tx_dropped;
1417                 uint64_t rx_bad_ip_csum;
1418                 uint64_t rx_bad_l4_csum;
1419                 uint64_t rx_bad_outer_l4_csum;
1420         } ports_stats[RTE_MAX_ETHPORTS];
1421         uint64_t total_rx_dropped = 0;
1422         uint64_t total_tx_dropped = 0;
1423         uint64_t total_rx_nombuf = 0;
1424         struct rte_eth_stats stats;
1425 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1426         uint64_t fwd_cycles = 0;
1427 #endif
1428         uint64_t total_recv = 0;
1429         uint64_t total_xmit = 0;
1430         struct rte_port *port;
1431         streamid_t sm_id;
1432         portid_t pt_id;
1433         int i;
1434
1435         memset(ports_stats, 0, sizeof(ports_stats));
1436
1437         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1438                 struct fwd_stream *fs = fwd_streams[sm_id];
1439
1440                 if (cur_fwd_config.nb_fwd_streams >
1441                     cur_fwd_config.nb_fwd_ports) {
1442                         fwd_stream_stats_display(sm_id);
1443                 } else {
1444                         ports_stats[fs->tx_port].tx_stream = fs;
1445                         ports_stats[fs->rx_port].rx_stream = fs;
1446                 }
1447
1448                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1449
1450                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1451                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1452                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1453                                 fs->rx_bad_outer_l4_csum;
1454
1455 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1456                 fwd_cycles += fs->core_cycles;
1457 #endif
1458         }
1459         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1460                 uint8_t j;
1461
1462                 pt_id = fwd_ports_ids[i];
1463                 port = &ports[pt_id];
1464
1465                 rte_eth_stats_get(pt_id, &stats);
1466                 stats.ipackets -= port->stats.ipackets;
1467                 stats.opackets -= port->stats.opackets;
1468                 stats.ibytes -= port->stats.ibytes;
1469                 stats.obytes -= port->stats.obytes;
1470                 stats.imissed -= port->stats.imissed;
1471                 stats.oerrors -= port->stats.oerrors;
1472                 stats.rx_nombuf -= port->stats.rx_nombuf;
1473
1474                 total_recv += stats.ipackets;
1475                 total_xmit += stats.opackets;
1476                 total_rx_dropped += stats.imissed;
1477                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1478                 total_tx_dropped += stats.oerrors;
1479                 total_rx_nombuf  += stats.rx_nombuf;
1480
1481                 printf("\n  %s Forward statistics for port %-2d %s\n",
1482                        fwd_stats_border, pt_id, fwd_stats_border);
1483
1484                 if (!port->rx_queue_stats_mapping_enabled &&
1485                     !port->tx_queue_stats_mapping_enabled) {
1486                         printf("  RX-packets: %-14"PRIu64
1487                                " RX-dropped: %-14"PRIu64
1488                                "RX-total: %-"PRIu64"\n",
1489                                stats.ipackets, stats.imissed,
1490                                stats.ipackets + stats.imissed);
1491
1492                         if (cur_fwd_eng == &csum_fwd_engine)
1493                                 printf("  Bad-ipcsum: %-14"PRIu64
1494                                        " Bad-l4csum: %-14"PRIu64
1495                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1496                                        ports_stats[pt_id].rx_bad_ip_csum,
1497                                        ports_stats[pt_id].rx_bad_l4_csum,
1498                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1499                         if (stats.ierrors + stats.rx_nombuf > 0) {
1500                                 printf("  RX-error: %-"PRIu64"\n",
1501                                        stats.ierrors);
1502                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1503                                        stats.rx_nombuf);
1504                         }
1505
1506                         printf("  TX-packets: %-14"PRIu64
1507                                " TX-dropped: %-14"PRIu64
1508                                "TX-total: %-"PRIu64"\n",
1509                                stats.opackets, ports_stats[pt_id].tx_dropped,
1510                                stats.opackets + ports_stats[pt_id].tx_dropped);
1511                 } else {
1512                         printf("  RX-packets:             %14"PRIu64
1513                                "    RX-dropped:%14"PRIu64
1514                                "    RX-total:%14"PRIu64"\n",
1515                                stats.ipackets, stats.imissed,
1516                                stats.ipackets + stats.imissed);
1517
1518                         if (cur_fwd_eng == &csum_fwd_engine)
1519                                 printf("  Bad-ipcsum:%14"PRIu64
1520                                        "    Bad-l4csum:%14"PRIu64
1521                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1522                                        ports_stats[pt_id].rx_bad_ip_csum,
1523                                        ports_stats[pt_id].rx_bad_l4_csum,
1524                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1525                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1526                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1527                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1528                                        stats.rx_nombuf);
1529                         }
1530
1531                         printf("  TX-packets:             %14"PRIu64
1532                                "    TX-dropped:%14"PRIu64
1533                                "    TX-total:%14"PRIu64"\n",
1534                                stats.opackets, ports_stats[pt_id].tx_dropped,
1535                                stats.opackets + ports_stats[pt_id].tx_dropped);
1536                 }
1537
1538 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1539                 if (ports_stats[pt_id].rx_stream)
1540                         pkt_burst_stats_display("RX",
1541                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1542                 if (ports_stats[pt_id].tx_stream)
1543                         pkt_burst_stats_display("TX",
1544                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1545 #endif
1546
1547                 if (port->rx_queue_stats_mapping_enabled) {
1548                         printf("\n");
1549                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1550                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1551                                        "     RX-errors:%14"PRIu64
1552                                        "    RX-bytes:%14"PRIu64"\n",
1553                                        j, stats.q_ipackets[j],
1554                                        stats.q_errors[j], stats.q_ibytes[j]);
1555                         }
1556                         printf("\n");
1557                 }
1558                 if (port->tx_queue_stats_mapping_enabled) {
1559                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1560                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1561                                        "                                 TX-bytes:%14"
1562                                        PRIu64"\n",
1563                                        j, stats.q_opackets[j],
1564                                        stats.q_obytes[j]);
1565                         }
1566                 }
1567
1568                 printf("  %s--------------------------------%s\n",
1569                        fwd_stats_border, fwd_stats_border);
1570         }
1571
1572         printf("\n  %s Accumulated forward statistics for all ports"
1573                "%s\n",
1574                acc_stats_border, acc_stats_border);
1575         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1576                "%-"PRIu64"\n"
1577                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1578                "%-"PRIu64"\n",
1579                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1580                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1581         if (total_rx_nombuf > 0)
1582                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1583         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1584                "%s\n",
1585                acc_stats_border, acc_stats_border);
1586 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1587         if (total_recv > 0)
1588                 printf("\n  CPU cycles/packet=%u (total cycles="
1589                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1590                        (unsigned int)(fwd_cycles / total_recv),
1591                        fwd_cycles, total_recv);
1592 #endif
1593 }
1594
1595 void
1596 fwd_stats_reset(void)
1597 {
1598         streamid_t sm_id;
1599         portid_t pt_id;
1600         int i;
1601
1602         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1603                 pt_id = fwd_ports_ids[i];
1604                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1605         }
1606         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1607                 struct fwd_stream *fs = fwd_streams[sm_id];
1608
1609                 fs->rx_packets = 0;
1610                 fs->tx_packets = 0;
1611                 fs->fwd_dropped = 0;
1612                 fs->rx_bad_ip_csum = 0;
1613                 fs->rx_bad_l4_csum = 0;
1614                 fs->rx_bad_outer_l4_csum = 0;
1615
1616 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1617                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1618                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1619 #endif
1620 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1621                 fs->core_cycles = 0;
1622 #endif
1623         }
1624 }
1625
1626 static void
1627 flush_fwd_rx_queues(void)
1628 {
1629         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1630         portid_t  rxp;
1631         portid_t port_id;
1632         queueid_t rxq;
1633         uint16_t  nb_rx;
1634         uint16_t  i;
1635         uint8_t   j;
1636         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1637         uint64_t timer_period;
1638
1639         /* convert to number of cycles */
1640         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1641
1642         for (j = 0; j < 2; j++) {
1643                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1644                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1645                                 port_id = fwd_ports_ids[rxp];
1646                                 /**
1647                                 * testpmd can stuck in the below do while loop
1648                                 * if rte_eth_rx_burst() always returns nonzero
1649                                 * packets. So timer is added to exit this loop
1650                                 * after 1sec timer expiry.
1651                                 */
1652                                 prev_tsc = rte_rdtsc();
1653                                 do {
1654                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1655                                                 pkts_burst, MAX_PKT_BURST);
1656                                         for (i = 0; i < nb_rx; i++)
1657                                                 rte_pktmbuf_free(pkts_burst[i]);
1658
1659                                         cur_tsc = rte_rdtsc();
1660                                         diff_tsc = cur_tsc - prev_tsc;
1661                                         timer_tsc += diff_tsc;
1662                                 } while ((nb_rx > 0) &&
1663                                         (timer_tsc < timer_period));
1664                                 timer_tsc = 0;
1665                         }
1666                 }
1667                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1668         }
1669 }
1670
1671 static void
1672 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1673 {
1674         struct fwd_stream **fsm;
1675         streamid_t nb_fs;
1676         streamid_t sm_id;
1677 #ifdef RTE_LIBRTE_BITRATE
1678         uint64_t tics_per_1sec;
1679         uint64_t tics_datum;
1680         uint64_t tics_current;
1681         uint16_t i, cnt_ports;
1682
1683         cnt_ports = nb_ports;
1684         tics_datum = rte_rdtsc();
1685         tics_per_1sec = rte_get_timer_hz();
1686 #endif
1687         fsm = &fwd_streams[fc->stream_idx];
1688         nb_fs = fc->stream_nb;
1689         do {
1690                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1691                         (*pkt_fwd)(fsm[sm_id]);
1692 #ifdef RTE_LIBRTE_BITRATE
1693                 if (bitrate_enabled != 0 &&
1694                                 bitrate_lcore_id == rte_lcore_id()) {
1695                         tics_current = rte_rdtsc();
1696                         if (tics_current - tics_datum >= tics_per_1sec) {
1697                                 /* Periodic bitrate calculation */
1698                                 for (i = 0; i < cnt_ports; i++)
1699                                         rte_stats_bitrate_calc(bitrate_data,
1700                                                 ports_ids[i]);
1701                                 tics_datum = tics_current;
1702                         }
1703                 }
1704 #endif
1705 #ifdef RTE_LIBRTE_LATENCY_STATS
1706                 if (latencystats_enabled != 0 &&
1707                                 latencystats_lcore_id == rte_lcore_id())
1708                         rte_latencystats_update();
1709 #endif
1710
1711         } while (! fc->stopped);
1712 }
1713
1714 static int
1715 start_pkt_forward_on_core(void *fwd_arg)
1716 {
1717         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1718                              cur_fwd_config.fwd_eng->packet_fwd);
1719         return 0;
1720 }
1721
1722 /*
1723  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1724  * Used to start communication flows in network loopback test configurations.
1725  */
1726 static int
1727 run_one_txonly_burst_on_core(void *fwd_arg)
1728 {
1729         struct fwd_lcore *fwd_lc;
1730         struct fwd_lcore tmp_lcore;
1731
1732         fwd_lc = (struct fwd_lcore *) fwd_arg;
1733         tmp_lcore = *fwd_lc;
1734         tmp_lcore.stopped = 1;
1735         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1736         return 0;
1737 }
1738
1739 /*
1740  * Launch packet forwarding:
1741  *     - Setup per-port forwarding context.
1742  *     - launch logical cores with their forwarding configuration.
1743  */
1744 static void
1745 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1746 {
1747         port_fwd_begin_t port_fwd_begin;
1748         unsigned int i;
1749         unsigned int lc_id;
1750         int diag;
1751
1752         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1753         if (port_fwd_begin != NULL) {
1754                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1755                         (*port_fwd_begin)(fwd_ports_ids[i]);
1756         }
1757         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1758                 lc_id = fwd_lcores_cpuids[i];
1759                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1760                         fwd_lcores[i]->stopped = 0;
1761                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1762                                                      fwd_lcores[i], lc_id);
1763                         if (diag != 0)
1764                                 printf("launch lcore %u failed - diag=%d\n",
1765                                        lc_id, diag);
1766                 }
1767         }
1768 }
1769
1770 /*
1771  * Launch packet forwarding configuration.
1772  */
1773 void
1774 start_packet_forwarding(int with_tx_first)
1775 {
1776         port_fwd_begin_t port_fwd_begin;
1777         port_fwd_end_t  port_fwd_end;
1778         struct rte_port *port;
1779         unsigned int i;
1780         portid_t   pt_id;
1781
1782         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1783                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1784
1785         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1786                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1787
1788         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1789                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1790                 (!nb_rxq || !nb_txq))
1791                 rte_exit(EXIT_FAILURE,
1792                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1793                         cur_fwd_eng->fwd_mode_name);
1794
1795         if (all_ports_started() == 0) {
1796                 printf("Not all ports were started\n");
1797                 return;
1798         }
1799         if (test_done == 0) {
1800                 printf("Packet forwarding already started\n");
1801                 return;
1802         }
1803
1804
1805         if(dcb_test) {
1806                 for (i = 0; i < nb_fwd_ports; i++) {
1807                         pt_id = fwd_ports_ids[i];
1808                         port = &ports[pt_id];
1809                         if (!port->dcb_flag) {
1810                                 printf("In DCB mode, all forwarding ports must "
1811                                        "be configured in this mode.\n");
1812                                 return;
1813                         }
1814                 }
1815                 if (nb_fwd_lcores == 1) {
1816                         printf("In DCB mode,the nb forwarding cores "
1817                                "should be larger than 1.\n");
1818                         return;
1819                 }
1820         }
1821         test_done = 0;
1822
1823         fwd_config_setup();
1824
1825         if(!no_flush_rx)
1826                 flush_fwd_rx_queues();
1827
1828         pkt_fwd_config_display(&cur_fwd_config);
1829         rxtx_config_display();
1830
1831         fwd_stats_reset();
1832         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1833                 pt_id = fwd_ports_ids[i];
1834                 port = &ports[pt_id];
1835                 map_port_queue_stats_mapping_registers(pt_id, port);
1836         }
1837         if (with_tx_first) {
1838                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1839                 if (port_fwd_begin != NULL) {
1840                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1841                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1842                 }
1843                 while (with_tx_first--) {
1844                         launch_packet_forwarding(
1845                                         run_one_txonly_burst_on_core);
1846                         rte_eal_mp_wait_lcore();
1847                 }
1848                 port_fwd_end = tx_only_engine.port_fwd_end;
1849                 if (port_fwd_end != NULL) {
1850                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1851                                 (*port_fwd_end)(fwd_ports_ids[i]);
1852                 }
1853         }
1854         launch_packet_forwarding(start_pkt_forward_on_core);
1855 }
1856
1857 void
1858 stop_packet_forwarding(void)
1859 {
1860         port_fwd_end_t port_fwd_end;
1861         lcoreid_t lc_id;
1862         portid_t pt_id;
1863         int i;
1864
1865         if (test_done) {
1866                 printf("Packet forwarding not started\n");
1867                 return;
1868         }
1869         printf("Telling cores to stop...");
1870         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1871                 fwd_lcores[lc_id]->stopped = 1;
1872         printf("\nWaiting for lcores to finish...\n");
1873         rte_eal_mp_wait_lcore();
1874         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1875         if (port_fwd_end != NULL) {
1876                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1877                         pt_id = fwd_ports_ids[i];
1878                         (*port_fwd_end)(pt_id);
1879                 }
1880         }
1881
1882         fwd_stats_display();
1883
1884         printf("\nDone.\n");
1885         test_done = 1;
1886 }
1887
1888 void
1889 dev_set_link_up(portid_t pid)
1890 {
1891         if (rte_eth_dev_set_link_up(pid) < 0)
1892                 printf("\nSet link up fail.\n");
1893 }
1894
1895 void
1896 dev_set_link_down(portid_t pid)
1897 {
1898         if (rte_eth_dev_set_link_down(pid) < 0)
1899                 printf("\nSet link down fail.\n");
1900 }
1901
1902 static int
1903 all_ports_started(void)
1904 {
1905         portid_t pi;
1906         struct rte_port *port;
1907
1908         RTE_ETH_FOREACH_DEV(pi) {
1909                 port = &ports[pi];
1910                 /* Check if there is a port which is not started */
1911                 if ((port->port_status != RTE_PORT_STARTED) &&
1912                         (port->slave_flag == 0))
1913                         return 0;
1914         }
1915
1916         /* No port is not started */
1917         return 1;
1918 }
1919
1920 int
1921 port_is_stopped(portid_t port_id)
1922 {
1923         struct rte_port *port = &ports[port_id];
1924
1925         if ((port->port_status != RTE_PORT_STOPPED) &&
1926             (port->slave_flag == 0))
1927                 return 0;
1928         return 1;
1929 }
1930
1931 int
1932 all_ports_stopped(void)
1933 {
1934         portid_t pi;
1935
1936         RTE_ETH_FOREACH_DEV(pi) {
1937                 if (!port_is_stopped(pi))
1938                         return 0;
1939         }
1940
1941         return 1;
1942 }
1943
1944 int
1945 port_is_started(portid_t port_id)
1946 {
1947         if (port_id_is_invalid(port_id, ENABLED_WARN))
1948                 return 0;
1949
1950         if (ports[port_id].port_status != RTE_PORT_STARTED)
1951                 return 0;
1952
1953         return 1;
1954 }
1955
1956 int
1957 start_port(portid_t pid)
1958 {
1959         int diag, need_check_link_status = -1;
1960         portid_t pi;
1961         queueid_t qi;
1962         struct rte_port *port;
1963         struct ether_addr mac_addr;
1964
1965         if (port_id_is_invalid(pid, ENABLED_WARN))
1966                 return 0;
1967
1968         if(dcb_config)
1969                 dcb_test = 1;
1970         RTE_ETH_FOREACH_DEV(pi) {
1971                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1972                         continue;
1973
1974                 need_check_link_status = 0;
1975                 port = &ports[pi];
1976                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1977                                                  RTE_PORT_HANDLING) == 0) {
1978                         printf("Port %d is now not stopped\n", pi);
1979                         continue;
1980                 }
1981
1982                 if (port->need_reconfig > 0) {
1983                         port->need_reconfig = 0;
1984
1985                         if (flow_isolate_all) {
1986                                 int ret = port_flow_isolate(pi, 1);
1987                                 if (ret) {
1988                                         printf("Failed to apply isolated"
1989                                                " mode on port %d\n", pi);
1990                                         return -1;
1991                                 }
1992                         }
1993                         configure_rxtx_dump_callbacks(0);
1994                         printf("Configuring Port %d (socket %u)\n", pi,
1995                                         port->socket_id);
1996                         /* configure port */
1997                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1998                                                 &(port->dev_conf));
1999                         if (diag != 0) {
2000                                 if (rte_atomic16_cmpset(&(port->port_status),
2001                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2002                                         printf("Port %d can not be set back "
2003                                                         "to stopped\n", pi);
2004                                 printf("Fail to configure port %d\n", pi);
2005                                 /* try to reconfigure port next time */
2006                                 port->need_reconfig = 1;
2007                                 return -1;
2008                         }
2009                 }
2010                 if (port->need_reconfig_queues > 0) {
2011                         port->need_reconfig_queues = 0;
2012                         /* setup tx queues */
2013                         for (qi = 0; qi < nb_txq; qi++) {
2014                                 if ((numa_support) &&
2015                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2016                                         diag = rte_eth_tx_queue_setup(pi, qi,
2017                                                 port->nb_tx_desc[qi],
2018                                                 txring_numa[pi],
2019                                                 &(port->tx_conf[qi]));
2020                                 else
2021                                         diag = rte_eth_tx_queue_setup(pi, qi,
2022                                                 port->nb_tx_desc[qi],
2023                                                 port->socket_id,
2024                                                 &(port->tx_conf[qi]));
2025
2026                                 if (diag == 0)
2027                                         continue;
2028
2029                                 /* Fail to setup tx queue, return */
2030                                 if (rte_atomic16_cmpset(&(port->port_status),
2031                                                         RTE_PORT_HANDLING,
2032                                                         RTE_PORT_STOPPED) == 0)
2033                                         printf("Port %d can not be set back "
2034                                                         "to stopped\n", pi);
2035                                 printf("Fail to configure port %d tx queues\n",
2036                                        pi);
2037                                 /* try to reconfigure queues next time */
2038                                 port->need_reconfig_queues = 1;
2039                                 return -1;
2040                         }
2041                         for (qi = 0; qi < nb_rxq; qi++) {
2042                                 /* setup rx queues */
2043                                 if ((numa_support) &&
2044                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2045                                         struct rte_mempool * mp =
2046                                                 mbuf_pool_find(rxring_numa[pi]);
2047                                         if (mp == NULL) {
2048                                                 printf("Failed to setup RX queue:"
2049                                                         "No mempool allocation"
2050                                                         " on the socket %d\n",
2051                                                         rxring_numa[pi]);
2052                                                 return -1;
2053                                         }
2054
2055                                         diag = rte_eth_rx_queue_setup(pi, qi,
2056                                              port->nb_rx_desc[qi],
2057                                              rxring_numa[pi],
2058                                              &(port->rx_conf[qi]),
2059                                              mp);
2060                                 } else {
2061                                         struct rte_mempool *mp =
2062                                                 mbuf_pool_find(port->socket_id);
2063                                         if (mp == NULL) {
2064                                                 printf("Failed to setup RX queue:"
2065                                                         "No mempool allocation"
2066                                                         " on the socket %d\n",
2067                                                         port->socket_id);
2068                                                 return -1;
2069                                         }
2070                                         diag = rte_eth_rx_queue_setup(pi, qi,
2071                                              port->nb_rx_desc[qi],
2072                                              port->socket_id,
2073                                              &(port->rx_conf[qi]),
2074                                              mp);
2075                                 }
2076                                 if (diag == 0)
2077                                         continue;
2078
2079                                 /* Fail to setup rx queue, return */
2080                                 if (rte_atomic16_cmpset(&(port->port_status),
2081                                                         RTE_PORT_HANDLING,
2082                                                         RTE_PORT_STOPPED) == 0)
2083                                         printf("Port %d can not be set back "
2084                                                         "to stopped\n", pi);
2085                                 printf("Fail to configure port %d rx queues\n",
2086                                        pi);
2087                                 /* try to reconfigure queues next time */
2088                                 port->need_reconfig_queues = 1;
2089                                 return -1;
2090                         }
2091                 }
2092                 configure_rxtx_dump_callbacks(verbose_level);
2093                 /* start port */
2094                 if (rte_eth_dev_start(pi) < 0) {
2095                         printf("Fail to start port %d\n", pi);
2096
2097                         /* Fail to setup rx queue, return */
2098                         if (rte_atomic16_cmpset(&(port->port_status),
2099                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2100                                 printf("Port %d can not be set back to "
2101                                                         "stopped\n", pi);
2102                         continue;
2103                 }
2104
2105                 if (rte_atomic16_cmpset(&(port->port_status),
2106                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2107                         printf("Port %d can not be set into started\n", pi);
2108
2109                 rte_eth_macaddr_get(pi, &mac_addr);
2110                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2111                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2112                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2113                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2114
2115                 /* at least one port started, need checking link status */
2116                 need_check_link_status = 1;
2117         }
2118
2119         if (need_check_link_status == 1 && !no_link_check)
2120                 check_all_ports_link_status(RTE_PORT_ALL);
2121         else if (need_check_link_status == 0)
2122                 printf("Please stop the ports first\n");
2123
2124         printf("Done\n");
2125         return 0;
2126 }
2127
2128 void
2129 stop_port(portid_t pid)
2130 {
2131         portid_t pi;
2132         struct rte_port *port;
2133         int need_check_link_status = 0;
2134
2135         if (dcb_test) {
2136                 dcb_test = 0;
2137                 dcb_config = 0;
2138         }
2139
2140         if (port_id_is_invalid(pid, ENABLED_WARN))
2141                 return;
2142
2143         printf("Stopping ports...\n");
2144
2145         RTE_ETH_FOREACH_DEV(pi) {
2146                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2147                         continue;
2148
2149                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2150                         printf("Please remove port %d from forwarding configuration.\n", pi);
2151                         continue;
2152                 }
2153
2154                 if (port_is_bonding_slave(pi)) {
2155                         printf("Please remove port %d from bonded device.\n", pi);
2156                         continue;
2157                 }
2158
2159                 port = &ports[pi];
2160                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2161                                                 RTE_PORT_HANDLING) == 0)
2162                         continue;
2163
2164                 rte_eth_dev_stop(pi);
2165
2166                 if (rte_atomic16_cmpset(&(port->port_status),
2167                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2168                         printf("Port %d can not be set into stopped\n", pi);
2169                 need_check_link_status = 1;
2170         }
2171         if (need_check_link_status && !no_link_check)
2172                 check_all_ports_link_status(RTE_PORT_ALL);
2173
2174         printf("Done\n");
2175 }
2176
2177 static void
2178 remove_invalid_ports_in(portid_t *array, portid_t *total)
2179 {
2180         portid_t i;
2181         portid_t new_total = 0;
2182
2183         for (i = 0; i < *total; i++)
2184                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2185                         array[new_total] = array[i];
2186                         new_total++;
2187                 }
2188         *total = new_total;
2189 }
2190
2191 static void
2192 remove_invalid_ports(void)
2193 {
2194         remove_invalid_ports_in(ports_ids, &nb_ports);
2195         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2196         nb_cfg_ports = nb_fwd_ports;
2197 }
2198
2199 void
2200 close_port(portid_t pid)
2201 {
2202         portid_t pi;
2203         struct rte_port *port;
2204
2205         if (port_id_is_invalid(pid, ENABLED_WARN))
2206                 return;
2207
2208         printf("Closing ports...\n");
2209
2210         RTE_ETH_FOREACH_DEV(pi) {
2211                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2212                         continue;
2213
2214                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2215                         printf("Please remove port %d from forwarding configuration.\n", pi);
2216                         continue;
2217                 }
2218
2219                 if (port_is_bonding_slave(pi)) {
2220                         printf("Please remove port %d from bonded device.\n", pi);
2221                         continue;
2222                 }
2223
2224                 port = &ports[pi];
2225                 if (rte_atomic16_cmpset(&(port->port_status),
2226                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2227                         printf("Port %d is already closed\n", pi);
2228                         continue;
2229                 }
2230
2231                 if (rte_atomic16_cmpset(&(port->port_status),
2232                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2233                         printf("Port %d is now not stopped\n", pi);
2234                         continue;
2235                 }
2236
2237                 if (port->flow_list)
2238                         port_flow_flush(pi);
2239                 rte_eth_dev_close(pi);
2240
2241                 remove_invalid_ports();
2242
2243                 if (rte_atomic16_cmpset(&(port->port_status),
2244                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2245                         printf("Port %d cannot be set to closed\n", pi);
2246         }
2247
2248         printf("Done\n");
2249 }
2250
2251 void
2252 reset_port(portid_t pid)
2253 {
2254         int diag;
2255         portid_t pi;
2256         struct rte_port *port;
2257
2258         if (port_id_is_invalid(pid, ENABLED_WARN))
2259                 return;
2260
2261         printf("Resetting ports...\n");
2262
2263         RTE_ETH_FOREACH_DEV(pi) {
2264                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2265                         continue;
2266
2267                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2268                         printf("Please remove port %d from forwarding "
2269                                "configuration.\n", pi);
2270                         continue;
2271                 }
2272
2273                 if (port_is_bonding_slave(pi)) {
2274                         printf("Please remove port %d from bonded device.\n",
2275                                pi);
2276                         continue;
2277                 }
2278
2279                 diag = rte_eth_dev_reset(pi);
2280                 if (diag == 0) {
2281                         port = &ports[pi];
2282                         port->need_reconfig = 1;
2283                         port->need_reconfig_queues = 1;
2284                 } else {
2285                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2286                 }
2287         }
2288
2289         printf("Done\n");
2290 }
2291
2292 void
2293 attach_port(char *identifier)
2294 {
2295         portid_t pi;
2296         struct rte_dev_iterator iterator;
2297
2298         printf("Attaching a new port...\n");
2299
2300         if (identifier == NULL) {
2301                 printf("Invalid parameters are specified\n");
2302                 return;
2303         }
2304
2305         if (rte_dev_probe(identifier) != 0) {
2306                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2307                 return;
2308         }
2309
2310         /* first attach mode: event */
2311         if (setup_on_probe_event) {
2312                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2313                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2314                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2315                                         ports[pi].need_setup != 0)
2316                                 setup_attached_port(pi);
2317                 return;
2318         }
2319
2320         /* second attach mode: iterator */
2321         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2322                 /* setup ports matching the devargs used for probing */
2323                 if (port_is_forwarding(pi))
2324                         continue; /* port was already attached before */
2325                 setup_attached_port(pi);
2326         }
2327 }
2328
2329 static void
2330 setup_attached_port(portid_t pi)
2331 {
2332         unsigned int socket_id;
2333
2334         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2335         /* if socket_id is invalid, set to the first available socket. */
2336         if (check_socket_id(socket_id) < 0)
2337                 socket_id = socket_ids[0];
2338         reconfig(pi, socket_id);
2339         rte_eth_promiscuous_enable(pi);
2340
2341         ports_ids[nb_ports++] = pi;
2342         fwd_ports_ids[nb_fwd_ports++] = pi;
2343         nb_cfg_ports = nb_fwd_ports;
2344         ports[pi].need_setup = 0;
2345         ports[pi].port_status = RTE_PORT_STOPPED;
2346
2347         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2348         printf("Done\n");
2349 }
2350
2351 void
2352 detach_port_device(portid_t port_id)
2353 {
2354         struct rte_device *dev;
2355         portid_t sibling;
2356
2357         printf("Removing a device...\n");
2358
2359         dev = rte_eth_devices[port_id].device;
2360         if (dev == NULL) {
2361                 printf("Device already removed\n");
2362                 return;
2363         }
2364
2365         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2366                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2367                         printf("Port not stopped\n");
2368                         return;
2369                 }
2370                 printf("Port was not closed\n");
2371                 if (ports[port_id].flow_list)
2372                         port_flow_flush(port_id);
2373         }
2374
2375         if (rte_dev_remove(dev) != 0) {
2376                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2377                 return;
2378         }
2379
2380         RTE_ETH_FOREACH_DEV_SIBLING(sibling, port_id) {
2381                 /* reset mapping between old ports and removed device */
2382                 rte_eth_devices[sibling].device = NULL;
2383                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2384                         /* sibling ports are forced to be closed */
2385                         ports[sibling].port_status = RTE_PORT_CLOSED;
2386                         printf("Port %u is closed\n", sibling);
2387                 }
2388         }
2389
2390         remove_invalid_ports();
2391
2392         printf("Device of port %u is detached\n", port_id);
2393         printf("Now total ports is %d\n", nb_ports);
2394         printf("Done\n");
2395         return;
2396 }
2397
2398 void
2399 pmd_test_exit(void)
2400 {
2401         struct rte_device *device;
2402         portid_t pt_id;
2403         int ret;
2404         int i;
2405
2406         if (test_done == 0)
2407                 stop_packet_forwarding();
2408
2409         if (ports != NULL) {
2410                 no_link_check = 1;
2411                 RTE_ETH_FOREACH_DEV(pt_id) {
2412                         printf("\nStopping port %d...\n", pt_id);
2413                         fflush(stdout);
2414                         stop_port(pt_id);
2415                 }
2416                 RTE_ETH_FOREACH_DEV(pt_id) {
2417                         printf("\nShutting down port %d...\n", pt_id);
2418                         fflush(stdout);
2419                         close_port(pt_id);
2420
2421                         /*
2422                          * This is a workaround to fix a virtio-user issue that
2423                          * requires to call clean-up routine to remove existing
2424                          * socket.
2425                          * This workaround valid only for testpmd, needs a fix
2426                          * valid for all applications.
2427                          * TODO: Implement proper resource cleanup
2428                          */
2429                         device = rte_eth_devices[pt_id].device;
2430                         if (device && !strcmp(device->driver->name, "net_virtio_user"))
2431                                 detach_port_device(pt_id);
2432                 }
2433         }
2434
2435         if (hot_plug) {
2436                 ret = rte_dev_event_monitor_stop();
2437                 if (ret) {
2438                         RTE_LOG(ERR, EAL,
2439                                 "fail to stop device event monitor.");
2440                         return;
2441                 }
2442
2443                 ret = rte_dev_event_callback_unregister(NULL,
2444                         dev_event_callback, NULL);
2445                 if (ret < 0) {
2446                         RTE_LOG(ERR, EAL,
2447                                 "fail to unregister device event callback.\n");
2448                         return;
2449                 }
2450
2451                 ret = rte_dev_hotplug_handle_disable();
2452                 if (ret) {
2453                         RTE_LOG(ERR, EAL,
2454                                 "fail to disable hotplug handling.\n");
2455                         return;
2456                 }
2457         }
2458         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2459                 if (mempools[i])
2460                         rte_mempool_free(mempools[i]);
2461         }
2462
2463         printf("\nBye...\n");
2464 }
2465
2466 typedef void (*cmd_func_t)(void);
2467 struct pmd_test_command {
2468         const char *cmd_name;
2469         cmd_func_t cmd_func;
2470 };
2471
2472 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2473
2474 /* Check the link status of all ports in up to 9s, and print them finally */
2475 static void
2476 check_all_ports_link_status(uint32_t port_mask)
2477 {
2478 #define CHECK_INTERVAL 100 /* 100ms */
2479 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2480         portid_t portid;
2481         uint8_t count, all_ports_up, print_flag = 0;
2482         struct rte_eth_link link;
2483
2484         printf("Checking link statuses...\n");
2485         fflush(stdout);
2486         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2487                 all_ports_up = 1;
2488                 RTE_ETH_FOREACH_DEV(portid) {
2489                         if ((port_mask & (1 << portid)) == 0)
2490                                 continue;
2491                         memset(&link, 0, sizeof(link));
2492                         rte_eth_link_get_nowait(portid, &link);
2493                         /* print link status if flag set */
2494                         if (print_flag == 1) {
2495                                 if (link.link_status)
2496                                         printf(
2497                                         "Port%d Link Up. speed %u Mbps- %s\n",
2498                                         portid, link.link_speed,
2499                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2500                                         ("full-duplex") : ("half-duplex\n"));
2501                                 else
2502                                         printf("Port %d Link Down\n", portid);
2503                                 continue;
2504                         }
2505                         /* clear all_ports_up flag if any link down */
2506                         if (link.link_status == ETH_LINK_DOWN) {
2507                                 all_ports_up = 0;
2508                                 break;
2509                         }
2510                 }
2511                 /* after finally printing all link status, get out */
2512                 if (print_flag == 1)
2513                         break;
2514
2515                 if (all_ports_up == 0) {
2516                         fflush(stdout);
2517                         rte_delay_ms(CHECK_INTERVAL);
2518                 }
2519
2520                 /* set the print_flag if all ports up or timeout */
2521                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2522                         print_flag = 1;
2523                 }
2524
2525                 if (lsc_interrupt)
2526                         break;
2527         }
2528 }
2529
2530 /*
2531  * This callback is for remove a port for a device. It has limitation because
2532  * it is not for multiple port removal for a device.
2533  * TODO: the device detach invoke will plan to be removed from user side to
2534  * eal. And convert all PMDs to free port resources on ether device closing.
2535  */
2536 static void
2537 rmv_port_callback(void *arg)
2538 {
2539         int need_to_start = 0;
2540         int org_no_link_check = no_link_check;
2541         portid_t port_id = (intptr_t)arg;
2542
2543         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2544
2545         if (!test_done && port_is_forwarding(port_id)) {
2546                 need_to_start = 1;
2547                 stop_packet_forwarding();
2548         }
2549         no_link_check = 1;
2550         stop_port(port_id);
2551         no_link_check = org_no_link_check;
2552         close_port(port_id);
2553         detach_port_device(port_id);
2554         if (need_to_start)
2555                 start_packet_forwarding(0);
2556 }
2557
2558 /* This function is used by the interrupt thread */
2559 static int
2560 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2561                   void *ret_param)
2562 {
2563         RTE_SET_USED(param);
2564         RTE_SET_USED(ret_param);
2565
2566         if (type >= RTE_ETH_EVENT_MAX) {
2567                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2568                         port_id, __func__, type);
2569                 fflush(stderr);
2570         } else if (event_print_mask & (UINT32_C(1) << type)) {
2571                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2572                         eth_event_desc[type]);
2573                 fflush(stdout);
2574         }
2575
2576         switch (type) {
2577         case RTE_ETH_EVENT_NEW:
2578                 ports[port_id].need_setup = 1;
2579                 ports[port_id].port_status = RTE_PORT_HANDLING;
2580                 break;
2581         case RTE_ETH_EVENT_INTR_RMV:
2582                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2583                         break;
2584                 if (rte_eal_alarm_set(100000,
2585                                 rmv_port_callback, (void *)(intptr_t)port_id))
2586                         fprintf(stderr, "Could not set up deferred device removal\n");
2587                 break;
2588         default:
2589                 break;
2590         }
2591         return 0;
2592 }
2593
2594 static int
2595 register_eth_event_callback(void)
2596 {
2597         int ret;
2598         enum rte_eth_event_type event;
2599
2600         for (event = RTE_ETH_EVENT_UNKNOWN;
2601                         event < RTE_ETH_EVENT_MAX; event++) {
2602                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2603                                 event,
2604                                 eth_event_callback,
2605                                 NULL);
2606                 if (ret != 0) {
2607                         TESTPMD_LOG(ERR, "Failed to register callback for "
2608                                         "%s event\n", eth_event_desc[event]);
2609                         return -1;
2610                 }
2611         }
2612
2613         return 0;
2614 }
2615
2616 /* This function is used by the interrupt thread */
2617 static void
2618 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2619                              __rte_unused void *arg)
2620 {
2621         uint16_t port_id;
2622         int ret;
2623
2624         if (type >= RTE_DEV_EVENT_MAX) {
2625                 fprintf(stderr, "%s called upon invalid event %d\n",
2626                         __func__, type);
2627                 fflush(stderr);
2628         }
2629
2630         switch (type) {
2631         case RTE_DEV_EVENT_REMOVE:
2632                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2633                         device_name);
2634                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2635                 if (ret) {
2636                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2637                                 device_name);
2638                         return;
2639                 }
2640                 /*
2641                  * Because the user's callback is invoked in eal interrupt
2642                  * callback, the interrupt callback need to be finished before
2643                  * it can be unregistered when detaching device. So finish
2644                  * callback soon and use a deferred removal to detach device
2645                  * is need. It is a workaround, once the device detaching be
2646                  * moved into the eal in the future, the deferred removal could
2647                  * be deleted.
2648                  */
2649                 if (rte_eal_alarm_set(100000,
2650                                 rmv_port_callback, (void *)(intptr_t)port_id))
2651                         RTE_LOG(ERR, EAL,
2652                                 "Could not set up deferred device removal\n");
2653                 break;
2654         case RTE_DEV_EVENT_ADD:
2655                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2656                         device_name);
2657                 /* TODO: After finish kernel driver binding,
2658                  * begin to attach port.
2659                  */
2660                 break;
2661         default:
2662                 break;
2663         }
2664 }
2665
2666 static int
2667 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2668 {
2669         uint16_t i;
2670         int diag;
2671         uint8_t mapping_found = 0;
2672
2673         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2674                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2675                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2676                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2677                                         tx_queue_stats_mappings[i].queue_id,
2678                                         tx_queue_stats_mappings[i].stats_counter_id);
2679                         if (diag != 0)
2680                                 return diag;
2681                         mapping_found = 1;
2682                 }
2683         }
2684         if (mapping_found)
2685                 port->tx_queue_stats_mapping_enabled = 1;
2686         return 0;
2687 }
2688
2689 static int
2690 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2691 {
2692         uint16_t i;
2693         int diag;
2694         uint8_t mapping_found = 0;
2695
2696         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2697                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2698                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2699                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2700                                         rx_queue_stats_mappings[i].queue_id,
2701                                         rx_queue_stats_mappings[i].stats_counter_id);
2702                         if (diag != 0)
2703                                 return diag;
2704                         mapping_found = 1;
2705                 }
2706         }
2707         if (mapping_found)
2708                 port->rx_queue_stats_mapping_enabled = 1;
2709         return 0;
2710 }
2711
2712 static void
2713 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2714 {
2715         int diag = 0;
2716
2717         diag = set_tx_queue_stats_mapping_registers(pi, port);
2718         if (diag != 0) {
2719                 if (diag == -ENOTSUP) {
2720                         port->tx_queue_stats_mapping_enabled = 0;
2721                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2722                 }
2723                 else
2724                         rte_exit(EXIT_FAILURE,
2725                                         "set_tx_queue_stats_mapping_registers "
2726                                         "failed for port id=%d diag=%d\n",
2727                                         pi, diag);
2728         }
2729
2730         diag = set_rx_queue_stats_mapping_registers(pi, port);
2731         if (diag != 0) {
2732                 if (diag == -ENOTSUP) {
2733                         port->rx_queue_stats_mapping_enabled = 0;
2734                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2735                 }
2736                 else
2737                         rte_exit(EXIT_FAILURE,
2738                                         "set_rx_queue_stats_mapping_registers "
2739                                         "failed for port id=%d diag=%d\n",
2740                                         pi, diag);
2741         }
2742 }
2743
2744 static void
2745 rxtx_port_config(struct rte_port *port)
2746 {
2747         uint16_t qid;
2748
2749         for (qid = 0; qid < nb_rxq; qid++) {
2750                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2751
2752                 /* Check if any Rx parameters have been passed */
2753                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2754                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2755
2756                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2757                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2758
2759                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2760                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2761
2762                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2763                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2764
2765                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2766                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2767
2768                 port->nb_rx_desc[qid] = nb_rxd;
2769         }
2770
2771         for (qid = 0; qid < nb_txq; qid++) {
2772                 port->tx_conf[qid] = port->dev_info.default_txconf;
2773
2774                 /* Check if any Tx parameters have been passed */
2775                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2776                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2777
2778                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2779                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2780
2781                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2782                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2783
2784                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2785                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2786
2787                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2788                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2789
2790                 port->nb_tx_desc[qid] = nb_txd;
2791         }
2792 }
2793
2794 void
2795 init_port_config(void)
2796 {
2797         portid_t pid;
2798         struct rte_port *port;
2799
2800         RTE_ETH_FOREACH_DEV(pid) {
2801                 port = &ports[pid];
2802                 port->dev_conf.fdir_conf = fdir_conf;
2803                 rte_eth_dev_info_get(pid, &port->dev_info);
2804                 if (nb_rxq > 1) {
2805                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2806                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2807                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2808                 } else {
2809                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2810                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2811                 }
2812
2813                 if (port->dcb_flag == 0) {
2814                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2815                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2816                         else
2817                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2818                 }
2819
2820                 rxtx_port_config(port);
2821
2822                 rte_eth_macaddr_get(pid, &port->eth_addr);
2823
2824                 map_port_queue_stats_mapping_registers(pid, port);
2825 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2826                 rte_pmd_ixgbe_bypass_init(pid);
2827 #endif
2828
2829                 if (lsc_interrupt &&
2830                     (rte_eth_devices[pid].data->dev_flags &
2831                      RTE_ETH_DEV_INTR_LSC))
2832                         port->dev_conf.intr_conf.lsc = 1;
2833                 if (rmv_interrupt &&
2834                     (rte_eth_devices[pid].data->dev_flags &
2835                      RTE_ETH_DEV_INTR_RMV))
2836                         port->dev_conf.intr_conf.rmv = 1;
2837         }
2838 }
2839
2840 void set_port_slave_flag(portid_t slave_pid)
2841 {
2842         struct rte_port *port;
2843
2844         port = &ports[slave_pid];
2845         port->slave_flag = 1;
2846 }
2847
2848 void clear_port_slave_flag(portid_t slave_pid)
2849 {
2850         struct rte_port *port;
2851
2852         port = &ports[slave_pid];
2853         port->slave_flag = 0;
2854 }
2855
2856 uint8_t port_is_bonding_slave(portid_t slave_pid)
2857 {
2858         struct rte_port *port;
2859
2860         port = &ports[slave_pid];
2861         if ((rte_eth_devices[slave_pid].data->dev_flags &
2862             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
2863                 return 1;
2864         return 0;
2865 }
2866
2867 const uint16_t vlan_tags[] = {
2868                 0,  1,  2,  3,  4,  5,  6,  7,
2869                 8,  9, 10, 11,  12, 13, 14, 15,
2870                 16, 17, 18, 19, 20, 21, 22, 23,
2871                 24, 25, 26, 27, 28, 29, 30, 31
2872 };
2873
2874 static  int
2875 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
2876                  enum dcb_mode_enable dcb_mode,
2877                  enum rte_eth_nb_tcs num_tcs,
2878                  uint8_t pfc_en)
2879 {
2880         uint8_t i;
2881         int32_t rc;
2882         struct rte_eth_rss_conf rss_conf;
2883
2884         /*
2885          * Builds up the correct configuration for dcb+vt based on the vlan tags array
2886          * given above, and the number of traffic classes available for use.
2887          */
2888         if (dcb_mode == DCB_VT_ENABLED) {
2889                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
2890                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
2891                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2892                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
2893
2894                 /* VMDQ+DCB RX and TX configurations */
2895                 vmdq_rx_conf->enable_default_pool = 0;
2896                 vmdq_rx_conf->default_pool = 0;
2897                 vmdq_rx_conf->nb_queue_pools =
2898                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2899                 vmdq_tx_conf->nb_queue_pools =
2900                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2901
2902                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
2903                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
2904                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
2905                         vmdq_rx_conf->pool_map[i].pools =
2906                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
2907                 }
2908                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2909                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
2910                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
2911                 }
2912
2913                 /* set DCB mode of RX and TX of multiple queues */
2914                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
2915                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
2916         } else {
2917                 struct rte_eth_dcb_rx_conf *rx_conf =
2918                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
2919                 struct rte_eth_dcb_tx_conf *tx_conf =
2920                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
2921
2922                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
2923                 if (rc != 0)
2924                         return rc;
2925
2926                 rx_conf->nb_tcs = num_tcs;
2927                 tx_conf->nb_tcs = num_tcs;
2928
2929                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2930                         rx_conf->dcb_tc[i] = i % num_tcs;
2931                         tx_conf->dcb_tc[i] = i % num_tcs;
2932                 }
2933
2934                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
2935                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
2936                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
2937         }
2938
2939         if (pfc_en)
2940                 eth_conf->dcb_capability_en =
2941                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
2942         else
2943                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
2944
2945         return 0;
2946 }
2947
2948 int
2949 init_port_dcb_config(portid_t pid,
2950                      enum dcb_mode_enable dcb_mode,
2951                      enum rte_eth_nb_tcs num_tcs,
2952                      uint8_t pfc_en)
2953 {
2954         struct rte_eth_conf port_conf;
2955         struct rte_port *rte_port;
2956         int retval;
2957         uint16_t i;
2958
2959         rte_port = &ports[pid];
2960
2961         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
2962         /* Enter DCB configuration status */
2963         dcb_config = 1;
2964
2965         port_conf.rxmode = rte_port->dev_conf.rxmode;
2966         port_conf.txmode = rte_port->dev_conf.txmode;
2967
2968         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
2969         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
2970         if (retval < 0)
2971                 return retval;
2972         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
2973
2974         /* re-configure the device . */
2975         rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
2976
2977         rte_eth_dev_info_get(pid, &rte_port->dev_info);
2978
2979         /* If dev_info.vmdq_pool_base is greater than 0,
2980          * the queue id of vmdq pools is started after pf queues.
2981          */
2982         if (dcb_mode == DCB_VT_ENABLED &&
2983             rte_port->dev_info.vmdq_pool_base > 0) {
2984                 printf("VMDQ_DCB multi-queue mode is nonsensical"
2985                         " for port %d.", pid);
2986                 return -1;
2987         }
2988
2989         /* Assume the ports in testpmd have the same dcb capability
2990          * and has the same number of rxq and txq in dcb mode
2991          */
2992         if (dcb_mode == DCB_VT_ENABLED) {
2993                 if (rte_port->dev_info.max_vfs > 0) {
2994                         nb_rxq = rte_port->dev_info.nb_rx_queues;
2995                         nb_txq = rte_port->dev_info.nb_tx_queues;
2996                 } else {
2997                         nb_rxq = rte_port->dev_info.max_rx_queues;
2998                         nb_txq = rte_port->dev_info.max_tx_queues;
2999                 }
3000         } else {
3001                 /*if vt is disabled, use all pf queues */
3002                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3003                         nb_rxq = rte_port->dev_info.max_rx_queues;
3004                         nb_txq = rte_port->dev_info.max_tx_queues;
3005                 } else {
3006                         nb_rxq = (queueid_t)num_tcs;
3007                         nb_txq = (queueid_t)num_tcs;
3008
3009                 }
3010         }
3011         rx_free_thresh = 64;
3012
3013         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3014
3015         rxtx_port_config(rte_port);
3016         /* VLAN filter */
3017         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3018         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3019                 rx_vft_set(pid, vlan_tags[i], 1);
3020
3021         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
3022         map_port_queue_stats_mapping_registers(pid, rte_port);
3023
3024         rte_port->dcb_flag = 1;
3025
3026         return 0;
3027 }
3028
3029 static void
3030 init_port(void)
3031 {
3032         /* Configuration of Ethernet ports. */
3033         ports = rte_zmalloc("testpmd: ports",
3034                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3035                             RTE_CACHE_LINE_SIZE);
3036         if (ports == NULL) {
3037                 rte_exit(EXIT_FAILURE,
3038                                 "rte_zmalloc(%d struct rte_port) failed\n",
3039                                 RTE_MAX_ETHPORTS);
3040         }
3041
3042         /* Initialize ports NUMA structures */
3043         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3044         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3045         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3046 }
3047
3048 static void
3049 force_quit(void)
3050 {
3051         pmd_test_exit();
3052         prompt_exit();
3053 }
3054
3055 static void
3056 print_stats(void)
3057 {
3058         uint8_t i;
3059         const char clr[] = { 27, '[', '2', 'J', '\0' };
3060         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3061
3062         /* Clear screen and move to top left */
3063         printf("%s%s", clr, top_left);
3064
3065         printf("\nPort statistics ====================================");
3066         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3067                 nic_stats_display(fwd_ports_ids[i]);
3068
3069         fflush(stdout);
3070 }
3071
3072 static void
3073 signal_handler(int signum)
3074 {
3075         if (signum == SIGINT || signum == SIGTERM) {
3076                 printf("\nSignal %d received, preparing to exit...\n",
3077                                 signum);
3078 #ifdef RTE_LIBRTE_PDUMP
3079                 /* uninitialize packet capture framework */
3080                 rte_pdump_uninit();
3081 #endif
3082 #ifdef RTE_LIBRTE_LATENCY_STATS
3083                 rte_latencystats_uninit();
3084 #endif
3085                 force_quit();
3086                 /* Set flag to indicate the force termination. */
3087                 f_quit = 1;
3088                 /* exit with the expected status */
3089                 signal(signum, SIG_DFL);
3090                 kill(getpid(), signum);
3091         }
3092 }
3093
3094 int
3095 main(int argc, char** argv)
3096 {
3097         int diag;
3098         portid_t port_id;
3099         uint16_t count;
3100         int ret;
3101
3102         signal(SIGINT, signal_handler);
3103         signal(SIGTERM, signal_handler);
3104
3105         diag = rte_eal_init(argc, argv);
3106         if (diag < 0)
3107                 rte_panic("Cannot init EAL\n");
3108
3109         testpmd_logtype = rte_log_register("testpmd");
3110         if (testpmd_logtype < 0)
3111                 rte_panic("Cannot register log type");
3112         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3113
3114         ret = register_eth_event_callback();
3115         if (ret != 0)
3116                 rte_panic("Cannot register for ethdev events");
3117
3118 #ifdef RTE_LIBRTE_PDUMP
3119         /* initialize packet capture framework */
3120         rte_pdump_init();
3121 #endif
3122
3123         count = 0;
3124         RTE_ETH_FOREACH_DEV(port_id) {
3125                 ports_ids[count] = port_id;
3126                 count++;
3127         }
3128         nb_ports = (portid_t) count;
3129         if (nb_ports == 0)
3130                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3131
3132         /* allocate port structures, and init them */
3133         init_port();
3134
3135         set_def_fwd_config();
3136         if (nb_lcores == 0)
3137                 rte_panic("Empty set of forwarding logical cores - check the "
3138                           "core mask supplied in the command parameters\n");
3139
3140         /* Bitrate/latency stats disabled by default */
3141 #ifdef RTE_LIBRTE_BITRATE
3142         bitrate_enabled = 0;
3143 #endif
3144 #ifdef RTE_LIBRTE_LATENCY_STATS
3145         latencystats_enabled = 0;
3146 #endif
3147
3148         /* on FreeBSD, mlockall() is disabled by default */
3149 #ifdef RTE_EXEC_ENV_FREEBSD
3150         do_mlockall = 0;
3151 #else
3152         do_mlockall = 1;
3153 #endif
3154
3155         argc -= diag;
3156         argv += diag;
3157         if (argc > 1)
3158                 launch_args_parse(argc, argv);
3159
3160         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3161                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3162                         strerror(errno));
3163         }
3164
3165         if (tx_first && interactive)
3166                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3167                                 "interactive mode.\n");
3168
3169         if (tx_first && lsc_interrupt) {
3170                 printf("Warning: lsc_interrupt needs to be off when "
3171                                 " using tx_first. Disabling.\n");
3172                 lsc_interrupt = 0;
3173         }
3174
3175         if (!nb_rxq && !nb_txq)
3176                 printf("Warning: Either rx or tx queues should be non-zero\n");
3177
3178         if (nb_rxq > 1 && nb_rxq > nb_txq)
3179                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3180                        "but nb_txq=%d will prevent to fully test it.\n",
3181                        nb_rxq, nb_txq);
3182
3183         init_config();
3184
3185         if (hot_plug) {
3186                 ret = rte_dev_hotplug_handle_enable();
3187                 if (ret) {
3188                         RTE_LOG(ERR, EAL,
3189                                 "fail to enable hotplug handling.");
3190                         return -1;
3191                 }
3192
3193                 ret = rte_dev_event_monitor_start();
3194                 if (ret) {
3195                         RTE_LOG(ERR, EAL,
3196                                 "fail to start device event monitoring.");
3197                         return -1;
3198                 }
3199
3200                 ret = rte_dev_event_callback_register(NULL,
3201                         dev_event_callback, NULL);
3202                 if (ret) {
3203                         RTE_LOG(ERR, EAL,
3204                                 "fail  to register device event callback\n");
3205                         return -1;
3206                 }
3207         }
3208
3209         if (start_port(RTE_PORT_ALL) != 0)
3210                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3211
3212         /* set all ports to promiscuous mode by default */
3213         RTE_ETH_FOREACH_DEV(port_id)
3214                 rte_eth_promiscuous_enable(port_id);
3215
3216         /* Init metrics library */
3217         rte_metrics_init(rte_socket_id());
3218
3219 #ifdef RTE_LIBRTE_LATENCY_STATS
3220         if (latencystats_enabled != 0) {
3221                 int ret = rte_latencystats_init(1, NULL);
3222                 if (ret)
3223                         printf("Warning: latencystats init()"
3224                                 " returned error %d\n", ret);
3225                 printf("Latencystats running on lcore %d\n",
3226                         latencystats_lcore_id);
3227         }
3228 #endif
3229
3230         /* Setup bitrate stats */
3231 #ifdef RTE_LIBRTE_BITRATE
3232         if (bitrate_enabled != 0) {
3233                 bitrate_data = rte_stats_bitrate_create();
3234                 if (bitrate_data == NULL)
3235                         rte_exit(EXIT_FAILURE,
3236                                 "Could not allocate bitrate data.\n");
3237                 rte_stats_bitrate_reg(bitrate_data);
3238         }
3239 #endif
3240
3241 #ifdef RTE_LIBRTE_CMDLINE
3242         if (strlen(cmdline_filename) != 0)
3243                 cmdline_read_from_file(cmdline_filename);
3244
3245         if (interactive == 1) {
3246                 if (auto_start) {
3247                         printf("Start automatic packet forwarding\n");
3248                         start_packet_forwarding(0);
3249                 }
3250                 prompt();
3251                 pmd_test_exit();
3252         } else
3253 #endif
3254         {
3255                 char c;
3256                 int rc;
3257
3258                 f_quit = 0;
3259
3260                 printf("No commandline core given, start packet forwarding\n");
3261                 start_packet_forwarding(tx_first);
3262                 if (stats_period != 0) {
3263                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3264                         uint64_t timer_period;
3265
3266                         /* Convert to number of cycles */
3267                         timer_period = stats_period * rte_get_timer_hz();
3268
3269                         while (f_quit == 0) {
3270                                 cur_time = rte_get_timer_cycles();
3271                                 diff_time += cur_time - prev_time;
3272
3273                                 if (diff_time >= timer_period) {
3274                                         print_stats();
3275                                         /* Reset the timer */
3276                                         diff_time = 0;
3277                                 }
3278                                 /* Sleep to avoid unnecessary checks */
3279                                 prev_time = cur_time;
3280                                 sleep(1);
3281                         }
3282                 }
3283
3284                 printf("Press enter to exit\n");
3285                 rc = read(0, &c, 1);
3286                 pmd_test_exit();
3287                 if (rc < 0)
3288                         return 1;
3289         }
3290
3291         return 0;
3292 }