net: add rte prefix to ether structures
[dpdk.git] / app / test-pmd / testpmd.c
index 8d584b0..2c736ec 100644 (file)
@@ -137,7 +137,7 @@ uint8_t txring_numa[RTE_MAX_ETHPORTS];
  * Must be instantiated with the ethernet addresses of peer traffic generator
  * ports.
  */
-struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
+struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 portid_t nb_peer_eth_addrs = 0;
 
 /*
@@ -188,6 +188,9 @@ struct fwd_engine * fwd_engines[] = {
        NULL,
 };
 
+struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
+uint16_t mempool_flags;
+
 struct fwd_config cur_fwd_config;
 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 uint32_t retry_enabled;
@@ -217,6 +220,9 @@ uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 /**< Split policy for packets to TX. */
 
+uint8_t txonly_multi_flow;
+/**< Whether multiple flows are generated in TXONLY mode. */
+
 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 
@@ -469,6 +475,7 @@ uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 struct vxlan_encap_conf vxlan_encap_conf = {
        .select_ipv4 = 1,
        .select_vlan = 0,
+       .select_tos_ttl = 0,
        .vni = "\x00\x00\x00",
        .udp_src = 0,
        .udp_dst = RTE_BE16(4789),
@@ -479,6 +486,8 @@ struct vxlan_encap_conf vxlan_encap_conf = {
        .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
                "\x00\x00\x00\x00\x00\x00\x11\x11",
        .vlan_tci = 0,
+       .ip_tos = 0,
+       .ip_ttl = 255,
        .eth_src = "\x00\x00\x00\x00\x00\x00",
        .eth_dst = "\xff\xff\xff\xff\xff\xff",
 };
@@ -825,11 +834,67 @@ setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 
        return 0;
 }
+static void
+dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
+            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
+{
+       uint16_t pid = 0;
+       int ret;
+
+       RTE_ETH_FOREACH_DEV(pid) {
+               struct rte_eth_dev *dev =
+                       &rte_eth_devices[pid];
+
+               ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
+                                       memhdr->len);
+               if (ret) {
+                       TESTPMD_LOG(DEBUG,
+                                   "unable to DMA unmap addr 0x%p "
+                                   "for device %s\n",
+                                   memhdr->addr, dev->data->name);
+               }
+       }
+       ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
+       if (ret) {
+               TESTPMD_LOG(DEBUG,
+                           "unable to un-register addr 0x%p\n", memhdr->addr);
+       }
+}
+
+static void
+dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
+          struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
+{
+       uint16_t pid = 0;
+       size_t page_size = sysconf(_SC_PAGESIZE);
+       int ret;
+
+       ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
+                                 page_size);
+       if (ret) {
+               TESTPMD_LOG(DEBUG,
+                           "unable to register addr 0x%p\n", memhdr->addr);
+               return;
+       }
+       RTE_ETH_FOREACH_DEV(pid) {
+               struct rte_eth_dev *dev =
+                       &rte_eth_devices[pid];
+
+               ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
+                                     memhdr->len);
+               if (ret) {
+                       TESTPMD_LOG(DEBUG,
+                                   "unable to DMA map addr 0x%p "
+                                   "for device %s\n",
+                                   memhdr->addr, dev->data->name);
+               }
+       }
+}
 
 /*
  * Configuration initialisation done once at init time.
  */
-static void
+static struct rte_mempool *
 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
                 unsigned int socket_id)
 {
@@ -859,7 +924,7 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
                        rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
                                mb_size, (unsigned int) mb_mempool_cache,
                                sizeof(struct rte_pktmbuf_pool_private),
-                               socket_id, 0);
+                               socket_id, mempool_flags);
                        if (rte_mp == NULL)
                                goto err;
 
@@ -870,6 +935,7 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
                        }
                        rte_pktmbuf_pool_init(rte_mp, NULL);
                        rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
+                       rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
                        break;
                }
        case MP_ALLOC_XMEM:
@@ -907,6 +973,7 @@ err:
        } else if (verbose_level > 0) {
                rte_mempool_dump(stdout, rte_mp);
        }
+       return rte_mp;
 }
 
 /*
@@ -1124,14 +1191,18 @@ init_config(void)
                uint8_t i;
 
                for (i = 0; i < num_sockets; i++)
-                       mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
-                                        socket_ids[i]);
+                       mempools[i] = mbuf_pool_create(mbuf_data_size,
+                                                      nb_mbuf_per_pool,
+                                                      socket_ids[i]);
        } else {
                if (socket_num == UMA_NO_CONFIG)
-                       mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0);
+                       mempools[0] = mbuf_pool_create(mbuf_data_size,
+                                                      nb_mbuf_per_pool, 0);
                else
-                       mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
-                                                socket_num);
+                       mempools[socket_num] = mbuf_pool_create
+                                                       (mbuf_data_size,
+                                                        nb_mbuf_per_pool,
+                                                        socket_num);
        }
 
        init_port_config();
@@ -1358,119 +1429,256 @@ pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
 
 static void
-fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats)
+fwd_stream_stats_display(streamid_t stream_id)
 {
-       struct rte_port *port;
-       uint8_t i;
+       struct fwd_stream *fs;
+       static const char *fwd_top_stats_border = "-------";
+
+       fs = fwd_streams[stream_id];
+       if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
+           (fs->fwd_dropped == 0))
+               return;
+       printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
+              "TX Port=%2d/Queue=%2d %s\n",
+              fwd_top_stats_border, fs->rx_port, fs->rx_queue,
+              fs->tx_port, fs->tx_queue, fwd_top_stats_border);
+       printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
+              " TX-dropped: %-14"PRIu64,
+              fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
+
+       /* if checksum mode */
+       if (cur_fwd_eng == &csum_fwd_engine) {
+               printf("  RX- bad IP checksum: %-14"PRIu64
+                      "  Rx- bad L4 checksum: %-14"PRIu64
+                      " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
+                       fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
+                       fs->rx_bad_outer_l4_csum);
+       } else {
+               printf("\n");
+       }
 
+#ifdef RTE_TEST_PMD_RECORD_BURST_STATS
+       pkt_burst_stats_display("RX", &fs->rx_burst_stats);
+       pkt_burst_stats_display("TX", &fs->tx_burst_stats);
+#endif
+}
+
+void
+fwd_stats_display(void)
+{
        static const char *fwd_stats_border = "----------------------";
+       static const char *acc_stats_border = "+++++++++++++++";
+       struct {
+               struct fwd_stream *rx_stream;
+               struct fwd_stream *tx_stream;
+               uint64_t tx_dropped;
+               uint64_t rx_bad_ip_csum;
+               uint64_t rx_bad_l4_csum;
+               uint64_t rx_bad_outer_l4_csum;
+       } ports_stats[RTE_MAX_ETHPORTS];
+       uint64_t total_rx_dropped = 0;
+       uint64_t total_tx_dropped = 0;
+       uint64_t total_rx_nombuf = 0;
+       struct rte_eth_stats stats;
+#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
+       uint64_t fwd_cycles = 0;
+#endif
+       uint64_t total_recv = 0;
+       uint64_t total_xmit = 0;
+       struct rte_port *port;
+       streamid_t sm_id;
+       portid_t pt_id;
+       int i;
 
-       port = &ports[port_id];
-       printf("\n  %s Forward statistics for port %-2d %s\n",
-              fwd_stats_border, port_id, fwd_stats_border);
+       memset(ports_stats, 0, sizeof(ports_stats));
 
-       if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) {
-               printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
-                      "%-"PRIu64"\n",
-                      stats->ipackets, stats->imissed,
-                      (uint64_t) (stats->ipackets + stats->imissed));
+       for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
+               struct fwd_stream *fs = fwd_streams[sm_id];
 
-               if (cur_fwd_eng == &csum_fwd_engine)
-                       printf("  Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64"Bad-outer-l4csum: %-14"PRIu64"\n",
-                              port->rx_bad_ip_csum, port->rx_bad_l4_csum,
-                              port->rx_bad_outer_l4_csum);
-               if ((stats->ierrors + stats->rx_nombuf) > 0) {
-                       printf("  RX-error: %-"PRIu64"\n",  stats->ierrors);
-                       printf("  RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf);
+               if (cur_fwd_config.nb_fwd_streams >
+                   cur_fwd_config.nb_fwd_ports) {
+                       fwd_stream_stats_display(sm_id);
+               } else {
+                       ports_stats[fs->tx_port].tx_stream = fs;
+                       ports_stats[fs->rx_port].rx_stream = fs;
                }
 
-               printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
-                      "%-"PRIu64"\n",
-                      stats->opackets, port->tx_dropped,
-                      (uint64_t) (stats->opackets + port->tx_dropped));
-       }
-       else {
-               printf("  RX-packets:             %14"PRIu64"    RX-dropped:%14"PRIu64"    RX-total:"
-                      "%14"PRIu64"\n",
-                      stats->ipackets, stats->imissed,
-                      (uint64_t) (stats->ipackets + stats->imissed));
+               ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
 
-               if (cur_fwd_eng == &csum_fwd_engine)
-                       printf("  Bad-ipcsum:%14"PRIu64"    Bad-l4csum:%14"PRIu64"    Bad-outer-l4csum: %-14"PRIu64"\n",
-                              port->rx_bad_ip_csum, port->rx_bad_l4_csum,
-                              port->rx_bad_outer_l4_csum);
-               if ((stats->ierrors + stats->rx_nombuf) > 0) {
-                       printf("  RX-error:%"PRIu64"\n", stats->ierrors);
-                       printf("  RX-nombufs:             %14"PRIu64"\n",
-                              stats->rx_nombuf);
-               }
+               ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
+               ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
+               ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
+                               fs->rx_bad_outer_l4_csum;
 
-               printf("  TX-packets:             %14"PRIu64"    TX-dropped:%14"PRIu64"    TX-total:"
-                      "%14"PRIu64"\n",
-                      stats->opackets, port->tx_dropped,
-                      (uint64_t) (stats->opackets + port->tx_dropped));
+#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
+               fwd_cycles += fs->core_cycles;
+#endif
        }
+       for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
+               uint8_t j;
+
+               pt_id = fwd_ports_ids[i];
+               port = &ports[pt_id];
+
+               rte_eth_stats_get(pt_id, &stats);
+               stats.ipackets -= port->stats.ipackets;
+               stats.opackets -= port->stats.opackets;
+               stats.ibytes -= port->stats.ibytes;
+               stats.obytes -= port->stats.obytes;
+               stats.imissed -= port->stats.imissed;
+               stats.oerrors -= port->stats.oerrors;
+               stats.rx_nombuf -= port->stats.rx_nombuf;
+
+               total_recv += stats.ipackets;
+               total_xmit += stats.opackets;
+               total_rx_dropped += stats.imissed;
+               total_tx_dropped += ports_stats[pt_id].tx_dropped;
+               total_tx_dropped += stats.oerrors;
+               total_rx_nombuf  += stats.rx_nombuf;
+
+               printf("\n  %s Forward statistics for port %-2d %s\n",
+                      fwd_stats_border, pt_id, fwd_stats_border);
+
+               if (!port->rx_queue_stats_mapping_enabled &&
+                   !port->tx_queue_stats_mapping_enabled) {
+                       printf("  RX-packets: %-14"PRIu64
+                              " RX-dropped: %-14"PRIu64
+                              "RX-total: %-"PRIu64"\n",
+                              stats.ipackets, stats.imissed,
+                              stats.ipackets + stats.imissed);
+
+                       if (cur_fwd_eng == &csum_fwd_engine)
+                               printf("  Bad-ipcsum: %-14"PRIu64
+                                      " Bad-l4csum: %-14"PRIu64
+                                      "Bad-outer-l4csum: %-14"PRIu64"\n",
+                                      ports_stats[pt_id].rx_bad_ip_csum,
+                                      ports_stats[pt_id].rx_bad_l4_csum,
+                                      ports_stats[pt_id].rx_bad_outer_l4_csum);
+                       if (stats.ierrors + stats.rx_nombuf > 0) {
+                               printf("  RX-error: %-"PRIu64"\n",
+                                      stats.ierrors);
+                               printf("  RX-nombufs: %-14"PRIu64"\n",
+                                      stats.rx_nombuf);
+                       }
+
+                       printf("  TX-packets: %-14"PRIu64
+                              " TX-dropped: %-14"PRIu64
+                              "TX-total: %-"PRIu64"\n",
+                              stats.opackets, ports_stats[pt_id].tx_dropped,
+                              stats.opackets + ports_stats[pt_id].tx_dropped);
+               } else {
+                       printf("  RX-packets:             %14"PRIu64
+                              "    RX-dropped:%14"PRIu64
+                              "    RX-total:%14"PRIu64"\n",
+                              stats.ipackets, stats.imissed,
+                              stats.ipackets + stats.imissed);
+
+                       if (cur_fwd_eng == &csum_fwd_engine)
+                               printf("  Bad-ipcsum:%14"PRIu64
+                                      "    Bad-l4csum:%14"PRIu64
+                                      "    Bad-outer-l4csum: %-14"PRIu64"\n",
+                                      ports_stats[pt_id].rx_bad_ip_csum,
+                                      ports_stats[pt_id].rx_bad_l4_csum,
+                                      ports_stats[pt_id].rx_bad_outer_l4_csum);
+                       if ((stats.ierrors + stats.rx_nombuf) > 0) {
+                               printf("  RX-error:%"PRIu64"\n", stats.ierrors);
+                               printf("  RX-nombufs:             %14"PRIu64"\n",
+                                      stats.rx_nombuf);
+                       }
+
+                       printf("  TX-packets:             %14"PRIu64
+                              "    TX-dropped:%14"PRIu64
+                              "    TX-total:%14"PRIu64"\n",
+                              stats.opackets, ports_stats[pt_id].tx_dropped,
+                              stats.opackets + ports_stats[pt_id].tx_dropped);
+               }
 
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
-       if (port->rx_stream)
-               pkt_burst_stats_display("RX",
-                       &port->rx_stream->rx_burst_stats);
-       if (port->tx_stream)
-               pkt_burst_stats_display("TX",
-                       &port->tx_stream->tx_burst_stats);
+               if (ports_stats[pt_id].rx_stream)
+                       pkt_burst_stats_display("RX",
+                               &ports_stats[pt_id].rx_stream->rx_burst_stats);
+               if (ports_stats[pt_id].tx_stream)
+                       pkt_burst_stats_display("TX",
+                               &ports_stats[pt_id].tx_stream->tx_burst_stats);
 #endif
 
-       if (port->rx_queue_stats_mapping_enabled) {
-               printf("\n");
-               for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
-                       printf("  Stats reg %2d RX-packets:%14"PRIu64
-                              "     RX-errors:%14"PRIu64
-                              "    RX-bytes:%14"PRIu64"\n",
-                              i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]);
-               }
-               printf("\n");
-       }
-       if (port->tx_queue_stats_mapping_enabled) {
-               for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
-                       printf("  Stats reg %2d TX-packets:%14"PRIu64
-                              "                                 TX-bytes:%14"PRIu64"\n",
-                              i, stats->q_opackets[i], stats->q_obytes[i]);
+               if (port->rx_queue_stats_mapping_enabled) {
+                       printf("\n");
+                       for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
+                               printf("  Stats reg %2d RX-packets:%14"PRIu64
+                                      "     RX-errors:%14"PRIu64
+                                      "    RX-bytes:%14"PRIu64"\n",
+                                      j, stats.q_ipackets[j],
+                                      stats.q_errors[j], stats.q_ibytes[j]);
+                       }
+                       printf("\n");
+               }
+               if (port->tx_queue_stats_mapping_enabled) {
+                       for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
+                               printf("  Stats reg %2d TX-packets:%14"PRIu64
+                                      "                                 TX-bytes:%14"
+                                      PRIu64"\n",
+                                      j, stats.q_opackets[j],
+                                      stats.q_obytes[j]);
+                       }
                }
+
+               printf("  %s--------------------------------%s\n",
+                      fwd_stats_border, fwd_stats_border);
        }
 
-       printf("  %s--------------------------------%s\n",
-              fwd_stats_border, fwd_stats_border);
+       printf("\n  %s Accumulated forward statistics for all ports"
+              "%s\n",
+              acc_stats_border, acc_stats_border);
+       printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
+              "%-"PRIu64"\n"
+              "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
+              "%-"PRIu64"\n",
+              total_recv, total_rx_dropped, total_recv + total_rx_dropped,
+              total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
+       if (total_rx_nombuf > 0)
+               printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
+       printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
+              "%s\n",
+              acc_stats_border, acc_stats_border);
+#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
+       if (total_recv > 0)
+               printf("\n  CPU cycles/packet=%u (total cycles="
+                      "%"PRIu64" / total RX packets=%"PRIu64")\n",
+                      (unsigned int)(fwd_cycles / total_recv),
+                      fwd_cycles, total_recv);
+#endif
 }
 
-static void
-fwd_stream_stats_display(streamid_t stream_id)
+void
+fwd_stats_reset(void)
 {
-       struct fwd_stream *fs;
-       static const char *fwd_top_stats_border = "-------";
-
-       fs = fwd_streams[stream_id];
-       if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
-           (fs->fwd_dropped == 0))
-               return;
-       printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
-              "TX Port=%2d/Queue=%2d %s\n",
-              fwd_top_stats_border, fs->rx_port, fs->rx_queue,
-              fs->tx_port, fs->tx_queue, fwd_top_stats_border);
-       printf("  RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u",
-              fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
+       streamid_t sm_id;
+       portid_t pt_id;
+       int i;
 
-       /* if checksum mode */
-       if (cur_fwd_eng == &csum_fwd_engine) {
-              printf("  RX- bad IP checksum: %-14u  Rx- bad L4 checksum: "
-                       "%-14u Rx- bad outer L4 checksum: %-14u\n",
-                       fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
-                       fs->rx_bad_outer_l4_csum);
+       for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
+               pt_id = fwd_ports_ids[i];
+               rte_eth_stats_get(pt_id, &ports[pt_id].stats);
        }
+       for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
+               struct fwd_stream *fs = fwd_streams[sm_id];
+
+               fs->rx_packets = 0;
+               fs->tx_packets = 0;
+               fs->fwd_dropped = 0;
+               fs->rx_bad_ip_csum = 0;
+               fs->rx_bad_l4_csum = 0;
+               fs->rx_bad_outer_l4_csum = 0;
 
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
-       pkt_burst_stats_display("RX", &fs->rx_burst_stats);
-       pkt_burst_stats_display("TX", &fs->tx_burst_stats);
+               memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
+               memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
 #endif
+#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
+               fs->core_cycles = 0;
+#endif
+       }
 }
 
 static void
@@ -1628,7 +1836,6 @@ start_packet_forwarding(int with_tx_first)
        struct rte_port *port;
        unsigned int i;
        portid_t   pt_id;
-       streamid_t sm_id;
 
        if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
                rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
@@ -1679,32 +1886,12 @@ start_packet_forwarding(int with_tx_first)
        pkt_fwd_config_display(&cur_fwd_config);
        rxtx_config_display();
 
+       fwd_stats_reset();
        for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
                pt_id = fwd_ports_ids[i];
                port = &ports[pt_id];
-               rte_eth_stats_get(pt_id, &port->stats);
-               port->tx_dropped = 0;
-
                map_port_queue_stats_mapping_registers(pt_id, port);
        }
-       for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
-               fwd_streams[sm_id]->rx_packets = 0;
-               fwd_streams[sm_id]->tx_packets = 0;
-               fwd_streams[sm_id]->fwd_dropped = 0;
-               fwd_streams[sm_id]->rx_bad_ip_csum = 0;
-               fwd_streams[sm_id]->rx_bad_l4_csum = 0;
-               fwd_streams[sm_id]->rx_bad_outer_l4_csum = 0;
-
-#ifdef RTE_TEST_PMD_RECORD_BURST_STATS
-               memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
-                      sizeof(fwd_streams[sm_id]->rx_burst_stats));
-               memset(&fwd_streams[sm_id]->tx_burst_stats, 0,
-                      sizeof(fwd_streams[sm_id]->tx_burst_stats));
-#endif
-#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
-               fwd_streams[sm_id]->core_cycles = 0;
-#endif
-       }
        if (with_tx_first) {
                port_fwd_begin = tx_only_engine.port_fwd_begin;
                if (port_fwd_begin != NULL) {
@@ -1728,26 +1915,10 @@ start_packet_forwarding(int with_tx_first)
 void
 stop_packet_forwarding(void)
 {
-       struct rte_eth_stats stats;
-       struct rte_port *port;
-       port_fwd_end_t  port_fwd_end;
+       port_fwd_end_t port_fwd_end;
+       lcoreid_t lc_id;
+       portid_t pt_id;
        int i;
-       portid_t   pt_id;
-       streamid_t sm_id;
-       lcoreid_t  lc_id;
-       uint64_t total_recv;
-       uint64_t total_xmit;
-       uint64_t total_rx_dropped;
-       uint64_t total_tx_dropped;
-       uint64_t total_rx_nombuf;
-       uint64_t tx_dropped;
-       uint64_t rx_bad_ip_csum;
-       uint64_t rx_bad_l4_csum;
-#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
-       uint64_t fwd_cycles;
-#endif
-
-       static const char *acc_stats_border = "+++++++++++++++";
 
        if (test_done) {
                printf("Packet forwarding not started\n");
@@ -1765,103 +1936,9 @@ stop_packet_forwarding(void)
                        (*port_fwd_end)(pt_id);
                }
        }
-#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
-       fwd_cycles = 0;
-#endif
-       for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
-               if (cur_fwd_config.nb_fwd_streams >
-                   cur_fwd_config.nb_fwd_ports) {
-                       fwd_stream_stats_display(sm_id);
-                       ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL;
-                       ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL;
-               } else {
-                       ports[fwd_streams[sm_id]->tx_port].tx_stream =
-                               fwd_streams[sm_id];
-                       ports[fwd_streams[sm_id]->rx_port].rx_stream =
-                               fwd_streams[sm_id];
-               }
-               tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped;
-               tx_dropped = (uint64_t) (tx_dropped +
-                                        fwd_streams[sm_id]->fwd_dropped);
-               ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped;
-
-               rx_bad_ip_csum =
-                       ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum;
-               rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum +
-                                        fwd_streams[sm_id]->rx_bad_ip_csum);
-               ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum =
-                                                       rx_bad_ip_csum;
-
-               rx_bad_l4_csum =
-                       ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum;
-               rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum +
-                                        fwd_streams[sm_id]->rx_bad_l4_csum);
-               ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum =
-                                                       rx_bad_l4_csum;
-
-               ports[fwd_streams[sm_id]->rx_port].rx_bad_outer_l4_csum +=
-                               fwd_streams[sm_id]->rx_bad_outer_l4_csum;
-
-#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
-               fwd_cycles = (uint64_t) (fwd_cycles +
-                                        fwd_streams[sm_id]->core_cycles);
-#endif
-       }
-       total_recv = 0;
-       total_xmit = 0;
-       total_rx_dropped = 0;
-       total_tx_dropped = 0;
-       total_rx_nombuf  = 0;
-       for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
-               pt_id = fwd_ports_ids[i];
-
-               port = &ports[pt_id];
-               rte_eth_stats_get(pt_id, &stats);
-               stats.ipackets -= port->stats.ipackets;
-               port->stats.ipackets = 0;
-               stats.opackets -= port->stats.opackets;
-               port->stats.opackets = 0;
-               stats.ibytes   -= port->stats.ibytes;
-               port->stats.ibytes = 0;
-               stats.obytes   -= port->stats.obytes;
-               port->stats.obytes = 0;
-               stats.imissed  -= port->stats.imissed;
-               port->stats.imissed = 0;
-               stats.oerrors  -= port->stats.oerrors;
-               port->stats.oerrors = 0;
-               stats.rx_nombuf -= port->stats.rx_nombuf;
-               port->stats.rx_nombuf = 0;
-
-               total_recv += stats.ipackets;
-               total_xmit += stats.opackets;
-               total_rx_dropped += stats.imissed;
-               total_tx_dropped += port->tx_dropped;
-               total_rx_nombuf  += stats.rx_nombuf;
 
-               fwd_port_stats_display(pt_id, &stats);
-       }
+       fwd_stats_display();
 
-       printf("\n  %s Accumulated forward statistics for all ports"
-              "%s\n",
-              acc_stats_border, acc_stats_border);
-       printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
-              "%-"PRIu64"\n"
-              "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
-              "%-"PRIu64"\n",
-              total_recv, total_rx_dropped, total_recv + total_rx_dropped,
-              total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
-       if (total_rx_nombuf > 0)
-               printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
-       printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
-              "%s\n",
-              acc_stats_border, acc_stats_border);
-#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
-       if (total_recv > 0)
-               printf("\n  CPU cycles/packet=%u (total cycles="
-                      "%"PRIu64" / total RX packets=%"PRIu64")\n",
-                      (unsigned int)(fwd_cycles / total_recv),
-                      fwd_cycles, total_recv);
-#endif
        printf("\nDone.\n");
        test_done = 1;
 }
@@ -1941,7 +2018,7 @@ start_port(portid_t pid)
        portid_t pi;
        queueid_t qi;
        struct rte_port *port;
-       struct ether_addr mac_addr;
+       struct rte_ether_addr mac_addr;
 
        if (port_id_is_invalid(pid, ENABLED_WARN))
                return 0;
@@ -2357,10 +2434,7 @@ detach_port_device(portid_t port_id)
                TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
                return;
        }
-
-       for (sibling = 0; sibling < RTE_MAX_ETHPORTS; sibling++) {
-               if (rte_eth_devices[sibling].device != dev)
-                       continue;
+       RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
                /* reset mapping between old ports and removed device */
                rte_eth_devices[sibling].device = NULL;
                if (ports[sibling].port_status != RTE_PORT_CLOSED) {
@@ -2384,16 +2458,28 @@ pmd_test_exit(void)
        struct rte_device *device;
        portid_t pt_id;
        int ret;
+       int i;
 
        if (test_done == 0)
                stop_packet_forwarding();
 
+       for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
+               if (mempools[i]) {
+                       if (mp_alloc_type == MP_ALLOC_ANON)
+                               rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
+                                                    NULL);
+               }
+       }
        if (ports != NULL) {
                no_link_check = 1;
                RTE_ETH_FOREACH_DEV(pt_id) {
-                       printf("\nShutting down port %d...\n", pt_id);
+                       printf("\nStopping port %d...\n", pt_id);
                        fflush(stdout);
                        stop_port(pt_id);
+               }
+               RTE_ETH_FOREACH_DEV(pt_id) {
+                       printf("\nShutting down port %d...\n", pt_id);
+                       fflush(stdout);
                        close_port(pt_id);
 
                        /*
@@ -2433,6 +2519,10 @@ pmd_test_exit(void)
                        return;
                }
        }
+       for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
+               if (mempools[i])
+                       rte_mempool_free(mempools[i]);
+       }
 
        printf("\nBye...\n");
 }
@@ -2719,9 +2809,12 @@ static void
 rxtx_port_config(struct rte_port *port)
 {
        uint16_t qid;
+       uint64_t offloads;
 
        for (qid = 0; qid < nb_rxq; qid++) {
+               offloads = port->rx_conf[qid].offloads;
                port->rx_conf[qid] = port->dev_info.default_rxconf;
+               port->rx_conf[qid].offloads |= offloads;
 
                /* Check if any Rx parameters have been passed */
                if (rx_pthresh != RTE_PMD_PARAM_UNSET)
@@ -2743,7 +2836,9 @@ rxtx_port_config(struct rte_port *port)
        }
 
        for (qid = 0; qid < nb_txq; qid++) {
+               offloads = port->tx_conf[qid].offloads;
                port->tx_conf[qid] = port->dev_info.default_txconf;
+               port->tx_conf[qid].offloads |= offloads;
 
                /* Check if any Tx parameters have been passed */
                if (tx_pthresh != RTE_PMD_PARAM_UNSET)
@@ -2946,8 +3041,9 @@ init_port_dcb_config(portid_t pid,
        port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
 
        /* re-configure the device . */
-       rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
-
+       retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
+       if (retval < 0)
+               return retval;
        rte_eth_dev_info_get(pid, &rte_port->dev_info);
 
        /* If dev_info.vmdq_pool_base is greater than 0,
@@ -3039,6 +3135,8 @@ print_stats(void)
        printf("\nPort statistics ====================================");
        for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
                nic_stats_display(fwd_ports_ids[i]);
+
+       fflush(stdout);
 }
 
 static void
@@ -3118,7 +3216,7 @@ main(int argc, char** argv)
 #endif
 
        /* on FreeBSD, mlockall() is disabled by default */
-#ifdef RTE_EXEC_ENV_BSDAPP
+#ifdef RTE_EXEC_ENV_FREEBSD
        do_mlockall = 0;
 #else
        do_mlockall = 1;