eal: do not hard-code install path with meson
[dpdk.git] / lib / librte_ether / rte_ethdev.c
index f8e6765..e9a999d 100644 (file)
@@ -1,34 +1,5 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2017 Intel Corporation
  */
 
 #include <sys/types.h>
@@ -47,7 +18,6 @@
 #include <rte_log.h>
 #include <rte_debug.h>
 #include <rte_interrupts.h>
-#include <rte_pci.h>
 #include <rte_memory.h>
 #include <rte_memcpy.h>
 #include <rte_memzone.h>
 #include <rte_errno.h>
 #include <rte_spinlock.h>
 #include <rte_string_fns.h>
+#include <rte_compat.h>
 
 #include "rte_ether.h"
 #include "rte_ethdev.h"
+#include "rte_ethdev_driver.h"
 #include "ethdev_profile.h"
 
 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
@@ -94,6 +66,7 @@ static const struct rte_eth_xstats_name_off rte_stats_strings[] = {
        {"tx_good_packets", offsetof(struct rte_eth_stats, opackets)},
        {"rx_good_bytes", offsetof(struct rte_eth_stats, ibytes)},
        {"tx_good_bytes", offsetof(struct rte_eth_stats, obytes)},
+       {"rx_missed_errors", offsetof(struct rte_eth_stats, imissed)},
        {"rx_errors", offsetof(struct rte_eth_stats, ierrors)},
        {"tx_errors", offsetof(struct rte_eth_stats, oerrors)},
        {"rx_mbuf_allocation_errors", offsetof(struct rte_eth_stats,
@@ -118,6 +91,61 @@ static const struct rte_eth_xstats_name_off rte_txq_stats_strings[] = {
 #define RTE_NB_TXQ_STATS (sizeof(rte_txq_stats_strings) /      \
                sizeof(rte_txq_stats_strings[0]))
 
+#define RTE_RX_OFFLOAD_BIT2STR(_name)  \
+       { DEV_RX_OFFLOAD_##_name, #_name }
+
+static const struct {
+       uint64_t offload;
+       const char *name;
+} rte_rx_offload_names[] = {
+       RTE_RX_OFFLOAD_BIT2STR(VLAN_STRIP),
+       RTE_RX_OFFLOAD_BIT2STR(IPV4_CKSUM),
+       RTE_RX_OFFLOAD_BIT2STR(UDP_CKSUM),
+       RTE_RX_OFFLOAD_BIT2STR(TCP_CKSUM),
+       RTE_RX_OFFLOAD_BIT2STR(TCP_LRO),
+       RTE_RX_OFFLOAD_BIT2STR(QINQ_STRIP),
+       RTE_RX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
+       RTE_RX_OFFLOAD_BIT2STR(MACSEC_STRIP),
+       RTE_RX_OFFLOAD_BIT2STR(HEADER_SPLIT),
+       RTE_RX_OFFLOAD_BIT2STR(VLAN_FILTER),
+       RTE_RX_OFFLOAD_BIT2STR(VLAN_EXTEND),
+       RTE_RX_OFFLOAD_BIT2STR(JUMBO_FRAME),
+       RTE_RX_OFFLOAD_BIT2STR(CRC_STRIP),
+       RTE_RX_OFFLOAD_BIT2STR(SCATTER),
+       RTE_RX_OFFLOAD_BIT2STR(TIMESTAMP),
+       RTE_RX_OFFLOAD_BIT2STR(SECURITY),
+};
+
+#undef RTE_RX_OFFLOAD_BIT2STR
+
+#define RTE_TX_OFFLOAD_BIT2STR(_name)  \
+       { DEV_TX_OFFLOAD_##_name, #_name }
+
+static const struct {
+       uint64_t offload;
+       const char *name;
+} rte_tx_offload_names[] = {
+       RTE_TX_OFFLOAD_BIT2STR(VLAN_INSERT),
+       RTE_TX_OFFLOAD_BIT2STR(IPV4_CKSUM),
+       RTE_TX_OFFLOAD_BIT2STR(UDP_CKSUM),
+       RTE_TX_OFFLOAD_BIT2STR(TCP_CKSUM),
+       RTE_TX_OFFLOAD_BIT2STR(SCTP_CKSUM),
+       RTE_TX_OFFLOAD_BIT2STR(TCP_TSO),
+       RTE_TX_OFFLOAD_BIT2STR(UDP_TSO),
+       RTE_TX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
+       RTE_TX_OFFLOAD_BIT2STR(QINQ_INSERT),
+       RTE_TX_OFFLOAD_BIT2STR(VXLAN_TNL_TSO),
+       RTE_TX_OFFLOAD_BIT2STR(GRE_TNL_TSO),
+       RTE_TX_OFFLOAD_BIT2STR(IPIP_TNL_TSO),
+       RTE_TX_OFFLOAD_BIT2STR(GENEVE_TNL_TSO),
+       RTE_TX_OFFLOAD_BIT2STR(MACSEC_INSERT),
+       RTE_TX_OFFLOAD_BIT2STR(MT_LOCKFREE),
+       RTE_TX_OFFLOAD_BIT2STR(MULTI_SEGS),
+       RTE_TX_OFFLOAD_BIT2STR(MBUF_FAST_FREE),
+       RTE_TX_OFFLOAD_BIT2STR(SECURITY),
+};
+
+#undef RTE_TX_OFFLOAD_BIT2STR
 
 /**
  * The user application callback description.
@@ -143,7 +171,8 @@ uint16_t
 rte_eth_find_next(uint16_t port_id)
 {
        while (port_id < RTE_MAX_ETHPORTS &&
-              rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED)
+              rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED &&
+              rte_eth_devices[port_id].state != RTE_ETH_DEV_REMOVED)
                port_id++;
 
        if (port_id >= RTE_MAX_ETHPORTS)
@@ -205,7 +234,6 @@ eth_dev_get(uint16_t port_id)
 
        eth_dev->data = &rte_eth_dev_data[port_id];
        eth_dev->state = RTE_ETH_DEV_ATTACHED;
-       TAILQ_INIT(&(eth_dev->link_intr_cbs));
 
        eth_dev_last_created_port = port_id;
 
@@ -239,6 +267,8 @@ rte_eth_dev_allocate(const char *name)
        eth_dev->data->port_id = port_id;
        eth_dev->data->mtu = ETHER_MTU;
 
+       _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_NEW, NULL);
+
        return eth_dev;
 }
 
@@ -280,6 +310,9 @@ rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
                return -EINVAL;
 
        eth_dev->state = RTE_ETH_DEV_UNUSED;
+
+       _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_DESTROY, NULL);
+
        return 0;
 }
 
@@ -287,8 +320,7 @@ int
 rte_eth_dev_is_valid_port(uint16_t port_id)
 {
        if (port_id >= RTE_MAX_ETHPORTS ||
-           (rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED &&
-            rte_eth_devices[port_id].state != RTE_ETH_DEV_DEFERRED))
+           (rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED))
                return 0;
        else
                return 1;
@@ -363,6 +395,16 @@ rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id)
        return -ENODEV;
 }
 
+static int
+eth_err(uint16_t port_id, int ret)
+{
+       if (ret == 0)
+               return 0;
+       if (rte_eth_dev_is_removed(port_id))
+               return -EIO;
+       return ret;
+}
+
 /* attach the new device, then store port_id of the device */
 int
 rte_eth_dev_attach(const char *devargs, uint16_t *port_id)
@@ -438,7 +480,7 @@ rte_eth_dev_detach(uint16_t port_id, char *name)
        if (ret < 0)
                goto err;
 
-       rte_eth_devices[port_id].state = RTE_ETH_DEV_UNUSED;
+       rte_eth_dev_release_port(&rte_eth_devices[port_id]);
        return 0;
 
 err:
@@ -517,7 +559,8 @@ rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id)
                return 0;
        }
 
-       return dev->dev_ops->rx_queue_start(dev, rx_queue_id);
+       return eth_err(port_id, dev->dev_ops->rx_queue_start(dev,
+                                                            rx_queue_id));
 
 }
 
@@ -543,7 +586,7 @@ rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id)
                return 0;
        }
 
-       return dev->dev_ops->rx_queue_stop(dev, rx_queue_id);
+       return eth_err(port_id, dev->dev_ops->rx_queue_stop(dev, rx_queue_id));
 
 }
 
@@ -569,7 +612,8 @@ rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id)
                return 0;
        }
 
-       return dev->dev_ops->tx_queue_start(dev, tx_queue_id);
+       return eth_err(port_id, dev->dev_ops->tx_queue_start(dev,
+                                                            tx_queue_id));
 
 }
 
@@ -595,7 +639,7 @@ rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id)
                return 0;
        }
 
-       return dev->dev_ops->tx_queue_stop(dev, tx_queue_id);
+       return eth_err(port_id, dev->dev_ops->tx_queue_stop(dev, tx_queue_id));
 
 }
 
@@ -771,6 +815,38 @@ rte_eth_convert_rx_offloads(const uint64_t rx_offloads,
                rxmode->security = 0;
 }
 
+const char * __rte_experimental
+rte_eth_dev_rx_offload_name(uint64_t offload)
+{
+       const char *name = "UNKNOWN";
+       unsigned int i;
+
+       for (i = 0; i < RTE_DIM(rte_rx_offload_names); ++i) {
+               if (offload == rte_rx_offload_names[i].offload) {
+                       name = rte_rx_offload_names[i].name;
+                       break;
+               }
+       }
+
+       return name;
+}
+
+const char * __rte_experimental
+rte_eth_dev_tx_offload_name(uint64_t offload)
+{
+       const char *name = "UNKNOWN";
+       unsigned int i;
+
+       for (i = 0; i < RTE_DIM(rte_tx_offload_names); ++i) {
+               if (offload == rte_tx_offload_names[i].offload) {
+                       name = rte_tx_offload_names[i].name;
+                       break;
+               }
+       }
+
+       return name;
+}
+
 int
 rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
                      const struct rte_eth_conf *dev_conf)
@@ -811,7 +887,7 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
         * Convert between the offloads API to enable PMDs to support
         * only one of them.
         */
-       if ((dev_conf->rxmode.ignore_offload_bitfield == 0)) {
+       if (dev_conf->rxmode.ignore_offload_bitfield == 0) {
                rte_eth_convert_rx_offload_bitfield(
                                &dev_conf->rxmode, &local_conf.rxmode.offloads);
        } else {
@@ -913,7 +989,7 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
                                port_id, diag);
                rte_eth_dev_rx_queue_config(dev, 0);
                rte_eth_dev_tx_queue_config(dev, 0);
-               return diag;
+               return eth_err(port_id, diag);
        }
 
        /* Initialize Rx profiling if enabled at compilation time. */
@@ -923,7 +999,7 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
                                port_id, diag);
                rte_eth_dev_rx_queue_config(dev, 0);
                rte_eth_dev_tx_queue_config(dev, 0);
-               return diag;
+               return eth_err(port_id, diag);
        }
 
        return 0;
@@ -1023,7 +1099,7 @@ rte_eth_dev_start(uint16_t port_id)
        if (diag == 0)
                dev->data->dev_started = 1;
        else
-               return diag;
+               return eth_err(port_id, diag);
 
        rte_eth_dev_config_restore(port_id);
 
@@ -1065,7 +1141,7 @@ rte_eth_dev_set_link_up(uint16_t port_id)
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_up, -ENOTSUP);
-       return (*dev->dev_ops->dev_set_link_up)(dev);
+       return eth_err(port_id, (*dev->dev_ops->dev_set_link_up)(dev));
 }
 
 int
@@ -1078,7 +1154,7 @@ rte_eth_dev_set_link_down(uint16_t port_id)
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_down, -ENOTSUP);
-       return (*dev->dev_ops->dev_set_link_down)(dev);
+       return eth_err(port_id, (*dev->dev_ops->dev_set_link_down)(dev));
 }
 
 void
@@ -1115,6 +1191,29 @@ rte_eth_dev_reset(uint16_t port_id)
        rte_eth_dev_stop(port_id);
        ret = dev->dev_ops->dev_reset(dev);
 
+       return eth_err(port_id, ret);
+}
+
+int __rte_experimental
+rte_eth_dev_is_removed(uint16_t port_id)
+{
+       struct rte_eth_dev *dev;
+       int ret;
+
+       RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
+
+       dev = &rte_eth_devices[port_id];
+
+       if (dev->state == RTE_ETH_DEV_REMOVED)
+               return 1;
+
+       RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->is_removed, 0);
+
+       ret = dev->dev_ops->is_removed(dev);
+       if (ret != 0)
+               /* Device is physically removed. */
+               dev->state = RTE_ETH_DEV_REMOVED;
+
        return ret;
 }
 
@@ -1217,7 +1316,7 @@ rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
                        dev->data->min_rx_buf_size = mbp_buf_size;
        }
 
-       return ret;
+       return eth_err(port_id, ret);
 }
 
 /**
@@ -1336,8 +1435,8 @@ rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
                                          &local_conf.offloads);
        }
 
-       return (*dev->dev_ops->tx_queue_setup)(dev, tx_queue_id, nb_tx_desc,
-                                              socket_id, &local_conf);
+       return eth_err(port_id, (*dev->dev_ops->tx_queue_setup)(dev,
+                      tx_queue_id, nb_tx_desc, socket_id, &local_conf));
 }
 
 void
@@ -1393,14 +1492,16 @@ int
 rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt)
 {
        struct rte_eth_dev *dev = &rte_eth_devices[port_id];
+       int ret;
 
        /* Validate Input Data. Bail if not valid or not supported. */
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_done_cleanup, -ENOTSUP);
 
        /* Call driver to free pending mbufs. */
-       return (*dev->dev_ops->tx_done_cleanup)(dev->data->tx_queues[queue_id],
-                       free_cnt);
+       ret = (*dev->dev_ops->tx_done_cleanup)(dev->data->tx_queues[queue_id],
+                                              free_cnt);
+       return eth_err(port_id, ret);
 }
 
 void
@@ -1537,7 +1638,7 @@ rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
        stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
-       return (*dev->dev_ops->stats_get)(dev, stats);
+       return eth_err(port_id, (*dev->dev_ops->stats_get)(dev, stats));
 }
 
 int
@@ -1583,12 +1684,12 @@ get_xstats_count(uint16_t port_id)
                count = (*dev->dev_ops->xstats_get_names_by_id)(dev, NULL,
                                NULL, 0);
                if (count < 0)
-                       return count;
+                       return eth_err(port_id, count);
        }
        if (dev->dev_ops->xstats_get_names != NULL) {
                count = (*dev->dev_ops->xstats_get_names)(dev, NULL, 0);
                if (count < 0)
-                       return count;
+                       return eth_err(port_id, count);
        } else
                count = 0;
 
@@ -1642,6 +1743,45 @@ rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
        return -EINVAL;
 }
 
+/* retrieve basic stats names */
+static int
+rte_eth_basic_stats_get_names(struct rte_eth_dev *dev,
+       struct rte_eth_xstat_name *xstats_names)
+{
+       int cnt_used_entries = 0;
+       uint32_t idx, id_queue;
+       uint16_t num_q;
+
+       for (idx = 0; idx < RTE_NB_STATS; idx++) {
+               snprintf(xstats_names[cnt_used_entries].name,
+                       sizeof(xstats_names[0].name),
+                       "%s", rte_stats_strings[idx].name);
+               cnt_used_entries++;
+       }
+       num_q = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
+       for (id_queue = 0; id_queue < num_q; id_queue++) {
+               for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) {
+                       snprintf(xstats_names[cnt_used_entries].name,
+                               sizeof(xstats_names[0].name),
+                               "rx_q%u%s",
+                               id_queue, rte_rxq_stats_strings[idx].name);
+                       cnt_used_entries++;
+               }
+
+       }
+       num_q = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
+       for (id_queue = 0; id_queue < num_q; id_queue++) {
+               for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) {
+                       snprintf(xstats_names[cnt_used_entries].name,
+                               sizeof(xstats_names[0].name),
+                               "tx_q%u%s",
+                               id_queue, rte_txq_stats_strings[idx].name);
+                       cnt_used_entries++;
+               }
+       }
+       return cnt_used_entries;
+}
+
 /* retrieve ethdev extended statistics names */
 int
 rte_eth_xstats_get_names_by_id(uint16_t port_id,
@@ -1650,7 +1790,9 @@ rte_eth_xstats_get_names_by_id(uint16_t port_id,
 {
        struct rte_eth_xstat_name *xstats_names_copy;
        unsigned int no_basic_stat_requested = 1;
+       unsigned int no_ext_stat_requested = 1;
        unsigned int expected_entries;
+       unsigned int basic_count;
        struct rte_eth_dev *dev;
        unsigned int i;
        int ret;
@@ -1658,6 +1800,7 @@ rte_eth_xstats_get_names_by_id(uint16_t port_id,
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
        dev = &rte_eth_devices[port_id];
 
+       basic_count = get_xstats_basic_count(dev);
        ret = get_xstats_count(port_id);
        if (ret < 0)
                return ret;
@@ -1675,7 +1818,6 @@ rte_eth_xstats_get_names_by_id(uint16_t port_id,
                return -EINVAL;
 
        if (ids && dev->dev_ops->xstats_get_names_by_id != NULL && size > 0) {
-               unsigned int basic_count = get_xstats_basic_count(dev);
                uint64_t ids_copy[size];
 
                for (i = 0; i < size; i++) {
@@ -1714,8 +1856,26 @@ rte_eth_xstats_get_names_by_id(uint16_t port_id,
                return -ENOMEM;
        }
 
+       if (ids) {
+               for (i = 0; i < size; i++) {
+                       if (ids[i] > basic_count) {
+                               no_ext_stat_requested = 0;
+                               break;
+                       }
+               }
+       }
+
        /* Fill xstats_names_copy structure */
-       rte_eth_xstats_get_names(port_id, xstats_names_copy, expected_entries);
+       if (ids && no_ext_stat_requested) {
+               rte_eth_basic_stats_get_names(dev, xstats_names_copy);
+       } else {
+               ret = rte_eth_xstats_get_names(port_id, xstats_names_copy,
+                       expected_entries);
+               if (ret < 0) {
+                       free(xstats_names_copy);
+                       return ret;
+               }
+       }
 
        /* Filter stats */
        for (i = 0; i < size; i++) {
@@ -1740,8 +1900,6 @@ rte_eth_xstats_get_names(uint16_t port_id,
        int cnt_used_entries;
        int cnt_expected_entries;
        int cnt_driver_entries;
-       uint32_t idx, id_queue;
-       uint16_t num_q;
 
        cnt_expected_entries = get_xstats_count(port_id);
        if (xstats_names == NULL || cnt_expected_entries < 0 ||
@@ -1750,35 +1908,9 @@ rte_eth_xstats_get_names(uint16_t port_id,
 
        /* port_id checked in get_xstats_count() */
        dev = &rte_eth_devices[port_id];
-       cnt_used_entries = 0;
 
-       for (idx = 0; idx < RTE_NB_STATS; idx++) {
-               snprintf(xstats_names[cnt_used_entries].name,
-                       sizeof(xstats_names[0].name),
-                       "%s", rte_stats_strings[idx].name);
-               cnt_used_entries++;
-       }
-       num_q = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
-       for (id_queue = 0; id_queue < num_q; id_queue++) {
-               for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) {
-                       snprintf(xstats_names[cnt_used_entries].name,
-                               sizeof(xstats_names[0].name),
-                               "rx_q%u%s",
-                               id_queue, rte_rxq_stats_strings[idx].name);
-                       cnt_used_entries++;
-               }
-
-       }
-       num_q = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
-       for (id_queue = 0; id_queue < num_q; id_queue++) {
-               for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) {
-                       snprintf(xstats_names[cnt_used_entries].name,
-                               sizeof(xstats_names[0].name),
-                               "tx_q%u%s",
-                               id_queue, rte_txq_stats_strings[idx].name);
-                       cnt_used_entries++;
-               }
-       }
+       cnt_used_entries = rte_eth_basic_stats_get_names(
+               dev, xstats_names);
 
        if (dev->dev_ops->xstats_get_names != NULL) {
                /* If there are any driver-specific xstats, append them
@@ -1789,29 +1921,87 @@ rte_eth_xstats_get_names(uint16_t port_id,
                        xstats_names + cnt_used_entries,
                        size - cnt_used_entries);
                if (cnt_driver_entries < 0)
-                       return cnt_driver_entries;
+                       return eth_err(port_id, cnt_driver_entries);
                cnt_used_entries += cnt_driver_entries;
        }
 
        return cnt_used_entries;
 }
 
+
+static int
+rte_eth_basic_stats_get(uint16_t port_id, struct rte_eth_xstat *xstats)
+{
+       struct rte_eth_dev *dev;
+       struct rte_eth_stats eth_stats;
+       unsigned int count = 0, i, q;
+       uint64_t val, *stats_ptr;
+       uint16_t nb_rxqs, nb_txqs;
+       int ret;
+
+       ret = rte_eth_stats_get(port_id, &eth_stats);
+       if (ret < 0)
+               return ret;
+
+       dev = &rte_eth_devices[port_id];
+
+       nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
+       nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
+
+       /* global stats */
+       for (i = 0; i < RTE_NB_STATS; i++) {
+               stats_ptr = RTE_PTR_ADD(&eth_stats,
+                                       rte_stats_strings[i].offset);
+               val = *stats_ptr;
+               xstats[count++].value = val;
+       }
+
+       /* per-rxq stats */
+       for (q = 0; q < nb_rxqs; q++) {
+               for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
+                       stats_ptr = RTE_PTR_ADD(&eth_stats,
+                                       rte_rxq_stats_strings[i].offset +
+                                       q * sizeof(uint64_t));
+                       val = *stats_ptr;
+                       xstats[count++].value = val;
+               }
+       }
+
+       /* per-txq stats */
+       for (q = 0; q < nb_txqs; q++) {
+               for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
+                       stats_ptr = RTE_PTR_ADD(&eth_stats,
+                                       rte_txq_stats_strings[i].offset +
+                                       q * sizeof(uint64_t));
+                       val = *stats_ptr;
+                       xstats[count++].value = val;
+               }
+       }
+       return count;
+}
+
 /* retrieve ethdev extended statistics */
 int
 rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids,
                         uint64_t *values, unsigned int size)
 {
        unsigned int no_basic_stat_requested = 1;
+       unsigned int no_ext_stat_requested = 1;
        unsigned int num_xstats_filled;
+       unsigned int basic_count;
        uint16_t expected_entries;
        struct rte_eth_dev *dev;
        unsigned int i;
        int ret;
 
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
-       expected_entries = get_xstats_count(port_id);
+       ret = get_xstats_count(port_id);
+       if (ret < 0)
+               return ret;
+       expected_entries = (uint16_t)ret;
        struct rte_eth_xstat xstats[expected_entries];
        dev = &rte_eth_devices[port_id];
+       basic_count = get_xstats_basic_count(dev);
 
        /* Return max number of stats if no ids given */
        if (!ids) {
@@ -1846,8 +2036,21 @@ rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids,
                                        values, size);
        }
 
+       if (ids) {
+               for (i = 0; i < size; i++) {
+                       if (ids[i] > basic_count) {
+                               no_ext_stat_requested = 0;
+                               break;
+                       }
+               }
+       }
+
        /* Fill the xstats structure */
-       ret = rte_eth_xstats_get(port_id, xstats, expected_entries);
+       if (ids && no_ext_stat_requested)
+               ret = rte_eth_basic_stats_get(port_id, xstats);
+       else
+               ret = rte_eth_xstats_get(port_id, xstats, expected_entries);
+
        if (ret < 0)
                return ret;
        num_xstats_filled = (unsigned int)ret;
@@ -1874,12 +2077,11 @@ int
 rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats,
        unsigned int n)
 {
-       struct rte_eth_stats eth_stats;
        struct rte_eth_dev *dev;
-       unsigned int count = 0, i, q;
+       unsigned int count = 0, i;
        signed int xcount = 0;
-       uint64_t val, *stats_ptr;
        uint16_t nb_rxqs, nb_txqs;
+       int ret;
 
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 
@@ -1902,45 +2104,17 @@ rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats,
                                     (n > count) ? n - count : 0);
 
                if (xcount < 0)
-                       return xcount;
+                       return eth_err(port_id, xcount);
        }
 
        if (n < count + xcount || xstats == NULL)
                return count + xcount;
 
        /* now fill the xstats structure */
-       count = 0;
-       rte_eth_stats_get(port_id, &eth_stats);
-
-       /* global stats */
-       for (i = 0; i < RTE_NB_STATS; i++) {
-               stats_ptr = RTE_PTR_ADD(&eth_stats,
-                                       rte_stats_strings[i].offset);
-               val = *stats_ptr;
-               xstats[count++].value = val;
-       }
-
-       /* per-rxq stats */
-       for (q = 0; q < nb_rxqs; q++) {
-               for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
-                       stats_ptr = RTE_PTR_ADD(&eth_stats,
-                                       rte_rxq_stats_strings[i].offset +
-                                       q * sizeof(uint64_t));
-                       val = *stats_ptr;
-                       xstats[count++].value = val;
-               }
-       }
-
-       /* per-txq stats */
-       for (q = 0; q < nb_txqs; q++) {
-               for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
-                       stats_ptr = RTE_PTR_ADD(&eth_stats,
-                                       rte_txq_stats_strings[i].offset +
-                                       q * sizeof(uint64_t));
-                       val = *stats_ptr;
-                       xstats[count++].value = val;
-               }
-       }
+       ret = rte_eth_basic_stats_get(port_id, xstats);
+       if (ret < 0)
+               return ret;
+       count = ret;
 
        for (i = 0; i < count; i++)
                xstats[i].id = i;
@@ -1990,8 +2164,8 @@ int
 rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id, uint16_t tx_queue_id,
                uint8_t stat_idx)
 {
-       return set_queue_stats_mapping(port_id, tx_queue_id, stat_idx,
-                       STAT_QMAP_TX);
+       return eth_err(port_id, set_queue_stats_mapping(port_id, tx_queue_id,
+                                               stat_idx, STAT_QMAP_TX));
 }
 
 
@@ -1999,8 +2173,8 @@ int
 rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id,
                uint8_t stat_idx)
 {
-       return set_queue_stats_mapping(port_id, rx_queue_id, stat_idx,
-                       STAT_QMAP_RX);
+       return eth_err(port_id, set_queue_stats_mapping(port_id, rx_queue_id,
+                                               stat_idx, STAT_QMAP_RX));
 }
 
 int
@@ -2012,7 +2186,8 @@ rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size)
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fw_version_get, -ENOTSUP);
-       return (*dev->dev_ops->fw_version_get)(dev, fw_version, fw_size);
+       return eth_err(port_id, (*dev->dev_ops->fw_version_get)(dev,
+                                                       fw_version, fw_size));
 }
 
 void
@@ -2102,7 +2277,7 @@ rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu)
        if (!ret)
                dev->data->mtu = mtu;
 
-       return ret;
+       return eth_err(port_id, ret);
 }
 
 int
@@ -2142,7 +2317,7 @@ rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on)
                        vfc->ids[vidx] &= ~(UINT64_C(1) << vbit);
        }
 
-       return ret;
+       return eth_err(port_id, ret);
 }
 
 int
@@ -2175,7 +2350,8 @@ rte_eth_dev_set_vlan_ether_type(uint16_t port_id,
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_tpid_set, -ENOTSUP);
 
-       return (*dev->dev_ops->vlan_tpid_set)(dev, vlan_type, tpid);
+       return eth_err(port_id, (*dev->dev_ops->vlan_tpid_set)(dev, vlan_type,
+                                                              tpid));
 }
 
 int
@@ -2185,10 +2361,14 @@ rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask)
        int ret = 0;
        int mask = 0;
        int cur, org = 0;
+       uint64_t orig_offloads;
 
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
        dev = &rte_eth_devices[port_id];
 
+       /* save original values in case of failure */
+       orig_offloads = dev->data->dev_conf.rxmode.offloads;
+
        /*check which option changed by application*/
        cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD);
        org = !!(dev->data->dev_conf.rxmode.offloads &
@@ -2241,9 +2421,15 @@ rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask)
         */
        rte_eth_convert_rx_offloads(dev->data->dev_conf.rxmode.offloads,
                                    &dev->data->dev_conf.rxmode);
-       (*dev->dev_ops->vlan_offload_set)(dev, mask);
+       ret = (*dev->dev_ops->vlan_offload_set)(dev, mask);
+       if (ret) {
+               /* hit an error restore  original values */
+               dev->data->dev_conf.rxmode.offloads = orig_offloads;
+               rte_eth_convert_rx_offloads(dev->data->dev_conf.rxmode.offloads,
+                                           &dev->data->dev_conf.rxmode);
+       }
 
-       return ret;
+       return eth_err(port_id, ret);
 }
 
 int
@@ -2278,9 +2464,8 @@ rte_eth_dev_set_vlan_pvid(uint16_t port_id, uint16_t pvid, int on)
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_pvid_set, -ENOTSUP);
-       (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on);
 
-       return 0;
+       return eth_err(port_id, (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on));
 }
 
 int
@@ -2292,7 +2477,7 @@ rte_eth_dev_flow_ctrl_get(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_get, -ENOTSUP);
        memset(fc_conf, 0, sizeof(*fc_conf));
-       return (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf);
+       return eth_err(port_id, (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf));
 }
 
 int
@@ -2308,7 +2493,7 @@ rte_eth_dev_flow_ctrl_set(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
 
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_set, -ENOTSUP);
-       return (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf);
+       return eth_err(port_id, (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf));
 }
 
 int
@@ -2326,7 +2511,8 @@ rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id,
        dev = &rte_eth_devices[port_id];
        /* High water, low water validation are device specific */
        if  (*dev->dev_ops->priority_flow_ctrl_set)
-               return (*dev->dev_ops->priority_flow_ctrl_set)(dev, pfc_conf);
+               return eth_err(port_id, (*dev->dev_ops->priority_flow_ctrl_set)
+                                       (dev, pfc_conf));
        return -ENOTSUP;
 }
 
@@ -2401,7 +2587,8 @@ rte_eth_dev_rss_reta_update(uint16_t port_id,
                return ret;
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_update, -ENOTSUP);
-       return (*dev->dev_ops->reta_update)(dev, reta_conf, reta_size);
+       return eth_err(port_id, (*dev->dev_ops->reta_update)(dev, reta_conf,
+                                                            reta_size));
 }
 
 int
@@ -2421,7 +2608,8 @@ rte_eth_dev_rss_reta_query(uint16_t port_id,
 
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_query, -ENOTSUP);
-       return (*dev->dev_ops->reta_query)(dev, reta_conf, reta_size);
+       return eth_err(port_id, (*dev->dev_ops->reta_query)(dev, reta_conf,
+                                                           reta_size));
 }
 
 int
@@ -2433,7 +2621,8 @@ rte_eth_dev_rss_hash_update(uint16_t port_id,
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
-       return (*dev->dev_ops->rss_hash_update)(dev, rss_conf);
+       return eth_err(port_id, (*dev->dev_ops->rss_hash_update)(dev,
+                                                                rss_conf));
 }
 
 int
@@ -2445,7 +2634,8 @@ rte_eth_dev_rss_hash_conf_get(uint16_t port_id,
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_conf_get, -ENOTSUP);
-       return (*dev->dev_ops->rss_hash_conf_get)(dev, rss_conf);
+       return eth_err(port_id, (*dev->dev_ops->rss_hash_conf_get)(dev,
+                                                                  rss_conf));
 }
 
 int
@@ -2467,7 +2657,8 @@ rte_eth_dev_udp_tunnel_port_add(uint16_t port_id,
 
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_add, -ENOTSUP);
-       return (*dev->dev_ops->udp_tunnel_port_add)(dev, udp_tunnel);
+       return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_add)(dev,
+                                                               udp_tunnel));
 }
 
 int
@@ -2490,7 +2681,8 @@ rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id,
        }
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_del, -ENOTSUP);
-       return (*dev->dev_ops->udp_tunnel_port_del)(dev, udp_tunnel);
+       return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_del)(dev,
+                                                               udp_tunnel));
 }
 
 int
@@ -2501,7 +2693,7 @@ rte_eth_led_on(uint16_t port_id)
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_on, -ENOTSUP);
-       return (*dev->dev_ops->dev_led_on)(dev);
+       return eth_err(port_id, (*dev->dev_ops->dev_led_on)(dev));
 }
 
 int
@@ -2512,7 +2704,7 @@ rte_eth_led_off(uint16_t port_id)
        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_off, -ENOTSUP);
-       return (*dev->dev_ops->dev_led_off)(dev);
+       return eth_err(port_id, (*dev->dev_ops->dev_led_off)(dev));
 }
 
 /*
@@ -2588,7 +2780,7 @@ rte_eth_dev_mac_addr_add(uint16_t port_id, struct ether_addr *addr,
                dev->data->mac_pool_sel[index] |= (1ULL << pool);
        }
 
-       return ret;
+       return eth_err(port_id, ret);
 }
 
 int
@@ -2684,7 +2876,7 @@ rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct ether_addr *addr,
 
        index = get_hash_mac_addr_index(port_id, addr);
        /* Check if it's already there, and do nothing */
-       if ((index >= 0) && (on))
+       if ((index >= 0) && on)
                return 0;
 
        if (index < 0) {
@@ -2714,7 +2906,7 @@ rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct ether_addr *addr,
                                        &dev->data->hash_mac_addrs[index]);
        }
 
-       return ret;
+       return eth_err(port_id, ret);
 }
 
 int
@@ -2727,7 +2919,8 @@ rte_eth_dev_uc_all_hash_table_set(uint16_t port_id, uint8_t on)
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_all_hash_table_set, -ENOTSUP);
-       return (*dev->dev_ops->uc_all_hash_table_set)(dev, on);
+       return eth_err(port_id, (*dev->dev_ops->uc_all_hash_table_set)(dev,
+                                                                      on));
 }
 
 int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
@@ -2757,7 +2950,8 @@ int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
        }
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_queue_rate_limit, -ENOTSUP);
-       return (*dev->dev_ops->set_queue_rate_limit)(dev, queue_idx, tx_rate);
+       return eth_err(port_id, (*dev->dev_ops->set_queue_rate_limit)(dev,
+                                                       queue_idx, tx_rate));
 }
 
 int
@@ -2795,7 +2989,8 @@ rte_eth_mirror_rule_set(uint16_t port_id,
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_set, -ENOTSUP);
 
-       return (*dev->dev_ops->mirror_rule_set)(dev, mirror_conf, rule_id, on);
+       return eth_err(port_id, (*dev->dev_ops->mirror_rule_set)(dev,
+                                               mirror_conf, rule_id, on));
 }
 
 int
@@ -2808,7 +3003,16 @@ rte_eth_mirror_rule_reset(uint16_t port_id, uint8_t rule_id)
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_reset, -ENOTSUP);
 
-       return (*dev->dev_ops->mirror_rule_reset)(dev, rule_id);
+       return eth_err(port_id, (*dev->dev_ops->mirror_rule_reset)(dev,
+                                                                  rule_id));
+}
+
+RTE_INIT(eth_dev_init_cb_lists)
+{
+       int i;
+
+       for (i = 0; i < RTE_MAX_ETHPORTS; i++)
+               TAILQ_INIT(&rte_eth_devices[i].link_intr_cbs);
 }
 
 int
@@ -2818,37 +3022,59 @@ rte_eth_dev_callback_register(uint16_t port_id,
 {
        struct rte_eth_dev *dev;
        struct rte_eth_dev_callback *user_cb;
+       uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
+       uint16_t last_port;
 
        if (!cb_fn)
                return -EINVAL;
 
-       RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
+       if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
+               RTE_LOG(ERR, EAL, "Invalid port_id=%d\n", port_id);
+               return -EINVAL;
+       }
+
+       if (port_id == RTE_ETH_ALL) {
+               next_port = 0;
+               last_port = RTE_MAX_ETHPORTS - 1;
+       } else {
+               next_port = last_port = port_id;
+       }
 
-       dev = &rte_eth_devices[port_id];
        rte_spinlock_lock(&rte_eth_dev_cb_lock);
 
-       TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
-               if (user_cb->cb_fn == cb_fn &&
-                       user_cb->cb_arg == cb_arg &&
-                       user_cb->event == event) {
-                       break;
+       do {
+               dev = &rte_eth_devices[next_port];
+
+               TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
+                       if (user_cb->cb_fn == cb_fn &&
+                               user_cb->cb_arg == cb_arg &&
+                               user_cb->event == event) {
+                               break;
+                       }
                }
-       }
 
-       /* create a new callback. */
-       if (user_cb == NULL) {
-               user_cb = rte_zmalloc("INTR_USER_CALLBACK",
-                                       sizeof(struct rte_eth_dev_callback), 0);
-               if (user_cb != NULL) {
-                       user_cb->cb_fn = cb_fn;
-                       user_cb->cb_arg = cb_arg;
-                       user_cb->event = event;
-                       TAILQ_INSERT_TAIL(&(dev->link_intr_cbs), user_cb, next);
+               /* create a new callback. */
+               if (user_cb == NULL) {
+                       user_cb = rte_zmalloc("INTR_USER_CALLBACK",
+                               sizeof(struct rte_eth_dev_callback), 0);
+                       if (user_cb != NULL) {
+                               user_cb->cb_fn = cb_fn;
+                               user_cb->cb_arg = cb_arg;
+                               user_cb->event = event;
+                               TAILQ_INSERT_TAIL(&(dev->link_intr_cbs),
+                                                 user_cb, next);
+                       } else {
+                               rte_spinlock_unlock(&rte_eth_dev_cb_lock);
+                               rte_eth_dev_callback_unregister(port_id, event,
+                                                               cb_fn, cb_arg);
+                               return -ENOMEM;
+                       }
+
                }
-       }
+       } while (++next_port <= last_port);
 
        rte_spinlock_unlock(&rte_eth_dev_cb_lock);
-       return (user_cb == NULL) ? -ENOMEM : 0;
+       return 0;
 }
 
 int
@@ -2859,36 +3085,50 @@ rte_eth_dev_callback_unregister(uint16_t port_id,
        int ret;
        struct rte_eth_dev *dev;
        struct rte_eth_dev_callback *cb, *next;
+       uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
+       uint16_t last_port;
 
        if (!cb_fn)
                return -EINVAL;
 
-       RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
+       if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
+               RTE_LOG(ERR, EAL, "Invalid port_id=%d\n", port_id);
+               return -EINVAL;
+       }
+
+       if (port_id == RTE_ETH_ALL) {
+               next_port = 0;
+               last_port = RTE_MAX_ETHPORTS - 1;
+       } else {
+               next_port = last_port = port_id;
+       }
 
-       dev = &rte_eth_devices[port_id];
        rte_spinlock_lock(&rte_eth_dev_cb_lock);
 
-       ret = 0;
-       for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL; cb = next) {
+       do {
+               dev = &rte_eth_devices[next_port];
+               ret = 0;
+               for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL;
+                    cb = next) {
 
-               next = TAILQ_NEXT(cb, next);
+                       next = TAILQ_NEXT(cb, next);
 
-               if (cb->cb_fn != cb_fn || cb->event != event ||
-                               (cb->cb_arg != (void *)-1 &&
-                               cb->cb_arg != cb_arg))
-                       continue;
+                       if (cb->cb_fn != cb_fn || cb->event != event ||
+                           (cb->cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
+                               continue;
 
-               /*
-                * if this callback is not executing right now,
-                * then remove it.
-                */
-               if (cb->active == 0) {
-                       TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
-                       rte_free(cb);
-               } else {
-                       ret = -EAGAIN;
+                       /*
+                        * if this callback is not executing right now,
+                        * then remove it.
+                        */
+                       if (cb->active == 0) {
+                               TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
+                               rte_free(cb);
+                       } else {
+                               ret = -EAGAIN;
+                       }
                }
-       }
+       } while (++next_port <= last_port);
 
        rte_spinlock_unlock(&rte_eth_dev_cb_lock);
        return ret;
@@ -2896,7 +3136,7 @@ rte_eth_dev_callback_unregister(uint16_t port_id,
 
 int
 _rte_eth_dev_callback_process(struct rte_eth_dev *dev,
-       enum rte_eth_event_type event, void *cb_arg, void *ret_param)
+       enum rte_eth_event_type event, void *ret_param)
 {
        struct rte_eth_dev_callback *cb_lst;
        struct rte_eth_dev_callback dev_cb;
@@ -2908,8 +3148,6 @@ _rte_eth_dev_callback_process(struct rte_eth_dev *dev,
                        continue;
                dev_cb = *cb_lst;
                cb_lst->active = 1;
-               if (cb_arg != NULL)
-                       dev_cb.cb_arg = cb_arg;
                if (ret_param != NULL)
                        dev_cb.ret_param = ret_param;
 
@@ -3030,7 +3268,8 @@ rte_eth_dev_rx_intr_enable(uint16_t port_id,
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_enable, -ENOTSUP);
-       return (*dev->dev_ops->rx_queue_intr_enable)(dev, queue_id);
+       return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_enable)(dev,
+                                                               queue_id));
 }
 
 int
@@ -3044,7 +3283,8 @@ rte_eth_dev_rx_intr_disable(uint16_t port_id,
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_disable, -ENOTSUP);
-       return (*dev->dev_ops->rx_queue_intr_disable)(dev, queue_id);
+       return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_disable)(dev,
+                                                               queue_id));
 }
 
 
@@ -3063,8 +3303,153 @@ rte_eth_dev_filter_supported(uint16_t port_id,
 }
 
 int
-rte_eth_dev_filter_ctrl(uint16_t port_id, enum rte_filter_type filter_type,
-                      enum rte_filter_op filter_op, void *arg)
+rte_eth_dev_filter_ctrl_v22(uint16_t port_id,
+                           enum rte_filter_type filter_type,
+                           enum rte_filter_op filter_op, void *arg);
+
+int
+rte_eth_dev_filter_ctrl_v22(uint16_t port_id,
+                           enum rte_filter_type filter_type,
+                           enum rte_filter_op filter_op, void *arg)
+{
+       struct rte_eth_fdir_info_v22 {
+               enum rte_fdir_mode mode;
+               struct rte_eth_fdir_masks mask;
+               struct rte_eth_fdir_flex_conf flex_conf;
+               uint32_t guarant_spc;
+               uint32_t best_spc;
+               uint32_t flow_types_mask[1];
+               uint32_t max_flexpayload;
+               uint32_t flex_payload_unit;
+               uint32_t max_flex_payload_segment_num;
+               uint16_t flex_payload_limit;
+               uint32_t flex_bitmask_unit;
+               uint32_t max_flex_bitmask_num;
+       };
+
+       struct rte_eth_hash_global_conf_v22 {
+               enum rte_eth_hash_function hash_func;
+               uint32_t sym_hash_enable_mask[1];
+               uint32_t valid_bit_mask[1];
+       };
+
+       struct rte_eth_hash_filter_info_v22 {
+               enum rte_eth_hash_filter_info_type info_type;
+               union {
+                       uint8_t enable;
+                       struct rte_eth_hash_global_conf_v22 global_conf;
+                       struct rte_eth_input_set_conf input_set_conf;
+               } info;
+       };
+
+       struct rte_eth_dev *dev;
+
+       RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+       dev = &rte_eth_devices[port_id];
+       RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
+       if (filter_op == RTE_ETH_FILTER_INFO) {
+               int retval;
+               struct rte_eth_fdir_info_v22 *fdir_info_v22;
+               struct rte_eth_fdir_info fdir_info;
+
+               fdir_info_v22 = (struct rte_eth_fdir_info_v22 *)arg;
+
+               retval = (*dev->dev_ops->filter_ctrl)(dev, filter_type,
+                         filter_op, (void *)&fdir_info);
+               fdir_info_v22->mode = fdir_info.mode;
+               fdir_info_v22->mask = fdir_info.mask;
+               fdir_info_v22->flex_conf = fdir_info.flex_conf;
+               fdir_info_v22->guarant_spc = fdir_info.guarant_spc;
+               fdir_info_v22->best_spc = fdir_info.best_spc;
+               fdir_info_v22->flow_types_mask[0] =
+                       (uint32_t)fdir_info.flow_types_mask[0];
+               fdir_info_v22->max_flexpayload = fdir_info.max_flexpayload;
+               fdir_info_v22->flex_payload_unit = fdir_info.flex_payload_unit;
+               fdir_info_v22->max_flex_payload_segment_num =
+                       fdir_info.max_flex_payload_segment_num;
+               fdir_info_v22->flex_payload_limit =
+                       fdir_info.flex_payload_limit;
+               fdir_info_v22->flex_bitmask_unit = fdir_info.flex_bitmask_unit;
+               fdir_info_v22->max_flex_bitmask_num =
+                       fdir_info.max_flex_bitmask_num;
+               return retval;
+       } else if (filter_op == RTE_ETH_FILTER_GET) {
+               int retval;
+               struct rte_eth_hash_filter_info f_info;
+               struct rte_eth_hash_filter_info_v22 *f_info_v22 =
+                       (struct rte_eth_hash_filter_info_v22 *)arg;
+
+               f_info.info_type = f_info_v22->info_type;
+               retval = (*dev->dev_ops->filter_ctrl)(dev, filter_type,
+                         filter_op, (void *)&f_info);
+
+               switch (f_info_v22->info_type) {
+               case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT:
+                       f_info_v22->info.enable = f_info.info.enable;
+                       break;
+               case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG:
+                       f_info_v22->info.global_conf.hash_func =
+                               f_info.info.global_conf.hash_func;
+                       f_info_v22->info.global_conf.sym_hash_enable_mask[0] =
+                               (uint32_t)
+                               f_info.info.global_conf.sym_hash_enable_mask[0];
+                       f_info_v22->info.global_conf.valid_bit_mask[0] =
+                               (uint32_t)
+                               f_info.info.global_conf.valid_bit_mask[0];
+                       break;
+               case RTE_ETH_HASH_FILTER_INPUT_SET_SELECT:
+                       f_info_v22->info.input_set_conf =
+                               f_info.info.input_set_conf;
+                       break;
+               default:
+                       break;
+               }
+               return retval;
+       } else if (filter_op == RTE_ETH_FILTER_SET) {
+               struct rte_eth_hash_filter_info f_info;
+               struct rte_eth_hash_filter_info_v22 *f_v22 =
+                       (struct rte_eth_hash_filter_info_v22 *)arg;
+
+               f_info.info_type = f_v22->info_type;
+               switch (f_v22->info_type) {
+               case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT:
+                       f_info.info.enable = f_v22->info.enable;
+                       break;
+               case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG:
+                       f_info.info.global_conf.hash_func =
+                               f_v22->info.global_conf.hash_func;
+                       f_info.info.global_conf.sym_hash_enable_mask[0] =
+                               (uint32_t)
+                               f_v22->info.global_conf.sym_hash_enable_mask[0];
+                       f_info.info.global_conf.valid_bit_mask[0] =
+                               (uint32_t)
+                               f_v22->info.global_conf.valid_bit_mask[0];
+                       break;
+               case RTE_ETH_HASH_FILTER_INPUT_SET_SELECT:
+                       f_info.info.input_set_conf =
+                               f_v22->info.input_set_conf;
+                       break;
+               default:
+                       break;
+               }
+               return (*dev->dev_ops->filter_ctrl)(dev, filter_type, filter_op,
+                                                   (void *)&f_info);
+       } else
+               return (*dev->dev_ops->filter_ctrl)(dev, filter_type, filter_op,
+                                                   arg);
+}
+VERSION_SYMBOL(rte_eth_dev_filter_ctrl, _v22, 2.2);
+
+int
+rte_eth_dev_filter_ctrl_v1802(uint16_t port_id,
+                             enum rte_filter_type filter_type,
+                             enum rte_filter_op filter_op, void *arg);
+
+int
+rte_eth_dev_filter_ctrl_v1802(uint16_t port_id,
+                             enum rte_filter_type filter_type,
+                             enum rte_filter_op filter_op, void *arg)
 {
        struct rte_eth_dev *dev;
 
@@ -3072,8 +3457,14 @@ rte_eth_dev_filter_ctrl(uint16_t port_id, enum rte_filter_type filter_type,
 
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
-       return (*dev->dev_ops->filter_ctrl)(dev, filter_type, filter_op, arg);
+       return eth_err(port_id, (*dev->dev_ops->filter_ctrl)(dev, filter_type,
+                                                            filter_op, arg));
 }
+BIND_DEFAULT_SYMBOL(rte_eth_dev_filter_ctrl, _v1802, 18.02);
+MAP_STATIC_SYMBOL(int rte_eth_dev_filter_ctrl(uint16_t port_id,
+                 enum rte_filter_type filter_type,
+                 enum rte_filter_op filter_op, void *arg),
+                 rte_eth_dev_filter_ctrl_v1802);
 
 void *
 rte_eth_add_rx_callback(uint16_t port_id, uint16_t queue_id,
@@ -3322,7 +3713,8 @@ rte_eth_dev_set_mc_addr_list(uint16_t port_id,
 
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_mc_addr_list, -ENOTSUP);
-       return dev->dev_ops->set_mc_addr_list(dev, mc_addr_set, nb_mc_addr);
+       return eth_err(port_id, dev->dev_ops->set_mc_addr_list(dev,
+                                               mc_addr_set, nb_mc_addr));
 }
 
 int
@@ -3334,7 +3726,7 @@ rte_eth_timesync_enable(uint16_t port_id)
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_enable, -ENOTSUP);
-       return (*dev->dev_ops->timesync_enable)(dev);
+       return eth_err(port_id, (*dev->dev_ops->timesync_enable)(dev));
 }
 
 int
@@ -3346,7 +3738,7 @@ rte_eth_timesync_disable(uint16_t port_id)
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_disable, -ENOTSUP);
-       return (*dev->dev_ops->timesync_disable)(dev);
+       return eth_err(port_id, (*dev->dev_ops->timesync_disable)(dev));
 }
 
 int
@@ -3359,7 +3751,8 @@ rte_eth_timesync_read_rx_timestamp(uint16_t port_id, struct timespec *timestamp,
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_rx_timestamp, -ENOTSUP);
-       return (*dev->dev_ops->timesync_read_rx_timestamp)(dev, timestamp, flags);
+       return eth_err(port_id, (*dev->dev_ops->timesync_read_rx_timestamp)
+                               (dev, timestamp, flags));
 }
 
 int
@@ -3372,7 +3765,8 @@ rte_eth_timesync_read_tx_timestamp(uint16_t port_id,
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_tx_timestamp, -ENOTSUP);
-       return (*dev->dev_ops->timesync_read_tx_timestamp)(dev, timestamp);
+       return eth_err(port_id, (*dev->dev_ops->timesync_read_tx_timestamp)
+                               (dev, timestamp));
 }
 
 int
@@ -3384,7 +3778,8 @@ rte_eth_timesync_adjust_time(uint16_t port_id, int64_t delta)
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_adjust_time, -ENOTSUP);
-       return (*dev->dev_ops->timesync_adjust_time)(dev, delta);
+       return eth_err(port_id, (*dev->dev_ops->timesync_adjust_time)(dev,
+                                                                     delta));
 }
 
 int
@@ -3396,7 +3791,8 @@ rte_eth_timesync_read_time(uint16_t port_id, struct timespec *timestamp)
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_time, -ENOTSUP);
-       return (*dev->dev_ops->timesync_read_time)(dev, timestamp);
+       return eth_err(port_id, (*dev->dev_ops->timesync_read_time)(dev,
+                                                               timestamp));
 }
 
 int
@@ -3408,7 +3804,8 @@ rte_eth_timesync_write_time(uint16_t port_id, const struct timespec *timestamp)
        dev = &rte_eth_devices[port_id];
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_write_time, -ENOTSUP);
-       return (*dev->dev_ops->timesync_write_time)(dev, timestamp);
+       return eth_err(port_id, (*dev->dev_ops->timesync_write_time)(dev,
+                                                               timestamp));
 }
 
 int
@@ -3420,7 +3817,7 @@ rte_eth_dev_get_reg_info(uint16_t port_id, struct rte_dev_reg_info *info)
 
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg, -ENOTSUP);
-       return (*dev->dev_ops->get_reg)(dev, info);
+       return eth_err(port_id, (*dev->dev_ops->get_reg)(dev, info));
 }
 
 int
@@ -3432,7 +3829,7 @@ rte_eth_dev_get_eeprom_length(uint16_t port_id)
 
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom_length, -ENOTSUP);
-       return (*dev->dev_ops->get_eeprom_length)(dev);
+       return eth_err(port_id, (*dev->dev_ops->get_eeprom_length)(dev));
 }
 
 int
@@ -3444,7 +3841,7 @@ rte_eth_dev_get_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
 
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom, -ENOTSUP);
-       return (*dev->dev_ops->get_eeprom)(dev, info);
+       return eth_err(port_id, (*dev->dev_ops->get_eeprom)(dev, info));
 }
 
 int
@@ -3456,7 +3853,7 @@ rte_eth_dev_set_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
 
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_eeprom, -ENOTSUP);
-       return (*dev->dev_ops->set_eeprom)(dev, info);
+       return eth_err(port_id, (*dev->dev_ops->set_eeprom)(dev, info));
 }
 
 int
@@ -3471,7 +3868,7 @@ rte_eth_dev_get_dcb_info(uint16_t port_id,
        memset(dcb_info, 0, sizeof(struct rte_eth_dcb_info));
 
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_dcb_info, -ENOTSUP);
-       return (*dev->dev_ops->get_dcb_info)(dev, dcb_info);
+       return eth_err(port_id, (*dev->dev_ops->get_dcb_info)(dev, dcb_info));
 }
 
 int
@@ -3494,7 +3891,8 @@ rte_eth_dev_l2_tunnel_eth_type_conf(uint16_t port_id,
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_eth_type_conf,
                                -ENOTSUP);
-       return (*dev->dev_ops->l2_tunnel_eth_type_conf)(dev, l2_tunnel);
+       return eth_err(port_id, (*dev->dev_ops->l2_tunnel_eth_type_conf)(dev,
+                                                               l2_tunnel));
 }
 
 int
@@ -3525,7 +3923,8 @@ rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id,
        dev = &rte_eth_devices[port_id];
        RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_offload_set,
                                -ENOTSUP);
-       return (*dev->dev_ops->l2_tunnel_offload_set)(dev, l2_tunnel, mask, en);
+       return eth_err(port_id, (*dev->dev_ops->l2_tunnel_offload_set)(dev,
+                                                       l2_tunnel, mask, en));
 }
 
 static void