X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fcxgbe%2Fcxgbe_ethdev.c;h=cdecf6b512ef0d4cc0df5f816c3daa4cd907fe3a;hb=dd4e429c95f90dfa049ad53cbd9e1142253f278c;hp=24c9a9323a50ea7b74a3b8fc81fc8201a92f4ba7;hpb=cd8c7c7ce241d2ea7c059a9df07caa9411ef19ed;p=dpdk.git

diff --git a/drivers/net/cxgbe/cxgbe_ethdev.c b/drivers/net/cxgbe/cxgbe_ethdev.c
index 24c9a9323a..cdecf6b512 100644
--- a/drivers/net/cxgbe/cxgbe_ethdev.c
+++ b/drivers/net/cxgbe/cxgbe_ethdev.c
@@ -21,21 +21,21 @@
 #include <rte_debug.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
-#include <rte_atomic.h>
 #include <rte_branch_prediction.h>
 #include <rte_memory.h>
 #include <rte_tailq.h>
 #include <rte_eal.h>
 #include <rte_alarm.h>
 #include <rte_ether.h>
-#include <rte_ethdev_driver.h>
-#include <rte_ethdev_pci.h>
+#include <ethdev_driver.h>
+#include <ethdev_pci.h>
 #include <rte_malloc.h>
 #include <rte_random.h>
 #include <rte_dev.h>
 
 #include "cxgbe.h"
 #include "cxgbe_pfvf.h"
+#include "cxgbe_flow.h"
 
 /*
  * Macros needed to support the PCI Device ID Table ...
@@ -56,20 +56,7 @@
 /*
  *... and the PCI ID Table itself ...
  */
-#include "t4_pci_id_tbl.h"
-
-#define CXGBE_TX_OFFLOADS (DEV_TX_OFFLOAD_VLAN_INSERT |\
-			   DEV_TX_OFFLOAD_IPV4_CKSUM |\
-			   DEV_TX_OFFLOAD_UDP_CKSUM |\
-			   DEV_TX_OFFLOAD_TCP_CKSUM |\
-			   DEV_TX_OFFLOAD_TCP_TSO)
-
-#define CXGBE_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_STRIP |\
-			   DEV_RX_OFFLOAD_CRC_STRIP |\
-			   DEV_RX_OFFLOAD_IPV4_CKSUM |\
-			   DEV_RX_OFFLOAD_JUMBO_FRAME |\
-			   DEV_RX_OFFLOAD_UDP_CKSUM |\
-			   DEV_RX_OFFLOAD_TCP_CKSUM)
+#include "base/t4_pci_id_tbl.h"
 
 uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 			 uint16_t nb_pkts)
@@ -77,20 +64,25 @@ uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	struct sge_eth_txq *txq = (struct sge_eth_txq *)tx_queue;
 	uint16_t pkts_sent, pkts_remain;
 	uint16_t total_sent = 0;
+	uint16_t idx = 0;
 	int ret = 0;
 
-	CXGBE_DEBUG_TX(adapter, "%s: txq = %p; tx_pkts = %p; nb_pkts = %d\n",
-		       __func__, txq, tx_pkts, nb_pkts);
-
 	t4_os_lock(&txq->txq_lock);
 	/* free up desc from already completed tx */
 	reclaim_completed_tx(&txq->q);
+	if (unlikely(!nb_pkts))
+		goto out_unlock;
+
+	rte_prefetch0(rte_pktmbuf_mtod(tx_pkts[0], volatile void *));
 	while (total_sent < nb_pkts) {
 		pkts_remain = nb_pkts - total_sent;
 
 		for (pkts_sent = 0; pkts_sent < pkts_remain; pkts_sent++) {
-			ret = t4_eth_xmit(txq, tx_pkts[total_sent + pkts_sent],
-					  nb_pkts);
+			idx = total_sent + pkts_sent;
+			if ((idx + 1) < nb_pkts)
+				rte_prefetch0(rte_pktmbuf_mtod(tx_pkts[idx + 1],
+							volatile void *));
+			ret = t4_eth_xmit(txq, tx_pkts[idx], nb_pkts);
 			if (ret < 0)
 				break;
 		}
@@ -101,6 +93,7 @@ uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		reclaim_completed_tx(&txq->q);
 	}
 
+out_unlock:
 	t4_os_unlock(&txq->txq_lock);
 	return total_sent;
 }
@@ -111,22 +104,17 @@ uint16_t cxgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 	struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)rx_queue;
 	unsigned int work_done;
 
-	CXGBE_DEBUG_RX(adapter, "%s: rxq->rspq.cntxt_id = %u; nb_pkts = %d\n",
-		       __func__, rxq->rspq.cntxt_id, nb_pkts);
-
 	if (cxgbe_poll(&rxq->rspq, rx_pkts, (unsigned int)nb_pkts, &work_done))
 		dev_err(adapter, "error in cxgbe poll\n");
 
-	CXGBE_DEBUG_RX(adapter, "%s: work_done = %u\n", __func__, work_done);
 	return work_done;
 }
 
-void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev,
+int cxgbe_dev_info_get(struct rte_eth_dev *eth_dev,
 			struct rte_eth_dev_info *device_info)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
-	int max_queues = adapter->sge.max_ethqsets / adapter->params.nports;
 
 	static const struct rte_eth_desc_lim cxgbe_desc_lim = {
 		.nb_max = CXGBE_MAX_RING_DESC_SIZE,
@@ -136,8 +124,8 @@ void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev,
 
 	device_info->min_rx_bufsize = CXGBE_MIN_RX_BUFSIZE;
 	device_info->max_rx_pktlen = CXGBE_MAX_RX_PKTLEN;
-	device_info->max_rx_queues = max_queues;
-	device_info->max_tx_queues = max_queues;
+	device_info->max_rx_queues = adapter->sge.max_ethqsets;
+	device_info->max_tx_queues = adapter->sge.max_ethqsets;
 	device_info->max_mac_addrs = 1;
 	/* XXX: For now we support one MAC/port */
 	device_info->max_vfs = adapter->params.arch.vfcount;
@@ -156,136 +144,218 @@ void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev,
 	device_info->rx_desc_lim = cxgbe_desc_lim;
 	device_info->tx_desc_lim = cxgbe_desc_lim;
 	cxgbe_get_speed_caps(pi, &device_info->speed_capa);
+
+	return 0;
 }
 
-void cxgbe_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
+int cxgbe_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
+	int ret;
+
+	if (adapter->params.rawf_size != 0) {
+		ret = cxgbe_mpstcam_rawf_enable(pi);
+		if (ret < 0)
+			return ret;
+	}
 
-	t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
-		      1, -1, 1, -1, false);
+	return t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
+			     1, -1, 1, -1, false);
 }
 
-void cxgbe_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
+int cxgbe_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
+	int ret;
+
+	if (adapter->params.rawf_size != 0) {
+		ret = cxgbe_mpstcam_rawf_disable(pi);
+		if (ret < 0)
+			return ret;
+	}
 
-	t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
-		      0, -1, 1, -1, false);
+	return t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
+			     0, -1, 1, -1, false);
 }
 
-void cxgbe_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
+int cxgbe_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 
 	/* TODO: address filters ?? */
 
-	t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
-		      -1, 1, 1, -1, false);
+	return t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
+			     -1, 1, 1, -1, false);
 }
 
-void cxgbe_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
+int cxgbe_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 
 	/* TODO: address filters ?? */
 
-	t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
-		      -1, 0, 1, -1, false);
+	return t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
+			     -1, 0, 1, -1, false);
 }
 
 int cxgbe_dev_link_update(struct rte_eth_dev *eth_dev,
-			  __rte_unused int wait_to_complete)
+			  int wait_to_complete)
+{
+	struct port_info *pi = eth_dev->data->dev_private;
+	unsigned int i, work_done, budget = 32;
+	struct link_config *lc = &pi->link_cfg;
+	struct adapter *adapter = pi->adapter;
+	struct rte_eth_link new_link = { 0 };
+	u8 old_link = pi->link_cfg.link_ok;
+	struct sge *s = &adapter->sge;
+
+	for (i = 0; i < CXGBE_LINK_STATUS_POLL_CNT; i++) {
+		if (!s->fw_evtq.desc)
+			break;
+
+		cxgbe_poll(&s->fw_evtq, NULL, budget, &work_done);
+
+		/* Exit if link status changed or always forced up */
+		if (pi->link_cfg.link_ok != old_link ||
+		    cxgbe_force_linkup(adapter))
+			break;
+
+		if (!wait_to_complete)
+			break;
+
+		rte_delay_ms(CXGBE_LINK_STATUS_POLL_MS);
+	}
+
+	new_link.link_status = cxgbe_force_linkup(adapter) ?
+			       ETH_LINK_UP : pi->link_cfg.link_ok;
+	new_link.link_autoneg = (lc->link_caps & FW_PORT_CAP32_ANEG) ? 1 : 0;
+	new_link.link_duplex = ETH_LINK_FULL_DUPLEX;
+	new_link.link_speed = t4_fwcap_to_speed(lc->link_caps);
+
+	return rte_eth_linkstatus_set(eth_dev, &new_link);
+}
+
+/**
+ * Set device link up.
+ */
+int cxgbe_dev_set_link_up(struct rte_eth_dev *dev)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
+	unsigned int work_done, budget = 32;
 	struct sge *s = &adapter->sge;
-	struct rte_eth_link *old_link = &eth_dev->data->dev_link;
-	unsigned int work_done, budget = 4;
+	int ret;
 
+	if (!s->fw_evtq.desc)
+		return -ENOMEM;
+
+	/* Flush all link events */
 	cxgbe_poll(&s->fw_evtq, NULL, budget, &work_done);
-	if (old_link->link_status == pi->link_cfg.link_ok)
-		return -1;  /* link not changed */
 
-	eth_dev->data->dev_link.link_status = pi->link_cfg.link_ok;
-	eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
-	eth_dev->data->dev_link.link_speed = pi->link_cfg.speed;
+	/* If link already up, nothing to do */
+	if (pi->link_cfg.link_ok)
+		return 0;
+
+	ret = cxgbe_set_link_status(pi, true);
+	if (ret)
+		return ret;
 
-	/* link has changed */
+	cxgbe_dev_link_update(dev, 1);
+	return 0;
+}
+
+/**
+ * Set device link down.
+ */
+int cxgbe_dev_set_link_down(struct rte_eth_dev *dev)
+{
+	struct port_info *pi = dev->data->dev_private;
+	struct adapter *adapter = pi->adapter;
+	unsigned int work_done, budget = 32;
+	struct sge *s = &adapter->sge;
+	int ret;
+
+	if (!s->fw_evtq.desc)
+		return -ENOMEM;
+
+	/* Flush all link events */
+	cxgbe_poll(&s->fw_evtq, NULL, budget, &work_done);
+
+	/* If link already down, nothing to do */
+	if (!pi->link_cfg.link_ok)
+		return 0;
+
+	ret = cxgbe_set_link_status(pi, false);
+	if (ret)
+		return ret;
+
+	cxgbe_dev_link_update(dev, 0);
 	return 0;
 }
 
 int cxgbe_dev_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 	struct rte_eth_dev_info dev_info;
 	int err;
-	uint16_t new_mtu = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
+	uint16_t new_mtu = mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
 
-	cxgbe_dev_info_get(eth_dev, &dev_info);
+	err = cxgbe_dev_info_get(eth_dev, &dev_info);
+	if (err != 0)
+		return err;
 
-	/* Must accommodate at least ETHER_MIN_MTU */
-	if ((new_mtu < ETHER_MIN_MTU) || (new_mtu > dev_info.max_rx_pktlen))
+	/* Must accommodate at least RTE_ETHER_MIN_MTU */
+	if (mtu < RTE_ETHER_MIN_MTU || new_mtu > dev_info.max_rx_pktlen)
 		return -EINVAL;
 
-	/* set to jumbo mode if needed */
-	if (new_mtu > ETHER_MAX_LEN)
-		eth_dev->data->dev_conf.rxmode.offloads |=
-			DEV_RX_OFFLOAD_JUMBO_FRAME;
-	else
-		eth_dev->data->dev_conf.rxmode.offloads &=
-			~DEV_RX_OFFLOAD_JUMBO_FRAME;
-
 	err = t4_set_rxmode(adapter, adapter->mbox, pi->viid, new_mtu, -1, -1,
 			    -1, -1, true);
-	if (!err)
-		eth_dev->data->dev_conf.rxmode.max_rx_pkt_len = new_mtu;
-
 	return err;
 }
 
 /*
  * Stop device.
  */
-void cxgbe_dev_close(struct rte_eth_dev *eth_dev)
+int cxgbe_dev_close(struct rte_eth_dev *eth_dev)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *temp_pi, *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
-	int i, dev_down = 0;
+	u8 i;
 
 	CXGBE_FUNC_TRACE();
 
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
 	if (!(adapter->flags & FULL_INIT_DONE))
-		return;
+		return 0;
+
+	if (!pi->viid)
+		return 0;
 
 	cxgbe_down(pi);
+	t4_sge_eth_release_queues(pi);
+	t4_free_vi(adapter, adapter->mbox, adapter->pf, 0, pi->viid);
+	pi->viid = 0;
 
-	/*
-	 *  We clear queues only if both tx and rx path of the port
-	 *  have been disabled
+	/* Free up the adapter-wide resources only after all the ports
+	 * under this PF have been closed.
 	 */
-	t4_sge_eth_clear_queues(pi);
-
-	/*  See if all ports are down */
 	for_each_port(adapter, i) {
-		pi = adap2pinfo(adapter, i);
-		/*
-		 * Skip first port of the adapter since it will be closed
-		 * by DPDK
-		 */
-		if (i == 0)
-			continue;
-		dev_down += (pi->eth_dev->data->dev_started == 0) ? 1 : 0;
+		temp_pi = adap2pinfo(adapter, i);
+		if (temp_pi->viid)
+			return 0;
 	}
 
-	/* If rest of the ports are stopped, then free up resources */
-	if (dev_down == (adapter->params.nports - 1))
-		cxgbe_close(adapter);
+	cxgbe_close(adapter);
+	rte_free(adapter);
+
+	return 0;
 }
 
 /* Start the device.
@@ -293,7 +363,8 @@ void cxgbe_dev_close(struct rte_eth_dev *eth_dev)
  */
 int cxgbe_dev_start(struct rte_eth_dev *eth_dev)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
+	struct rte_eth_rxmode *rx_conf = &eth_dev->data->dev_conf.rxmode;
 	struct adapter *adapter = pi->adapter;
 	int err = 0, i;
 
@@ -314,9 +385,14 @@ int cxgbe_dev_start(struct rte_eth_dev *eth_dev)
 			goto out;
 	}
 
+	if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER)
+		eth_dev->data->scattered_rx = 1;
+	else
+		eth_dev->data->scattered_rx = 0;
+
 	cxgbe_enable_rx_queues(pi);
 
-	err = setup_rss(pi);
+	err = cxgbe_setup_rss(pi);
 	if (err)
 		goto out;
 
@@ -332,7 +408,7 @@ int cxgbe_dev_start(struct rte_eth_dev *eth_dev)
 			goto out;
 	}
 
-	err = link_start(pi);
+	err = cxgbe_link_start(pi);
 	if (err)
 		goto out;
 
@@ -343,15 +419,15 @@ out:
 /*
  * Stop device: disable rx and tx functions to allow for reconfiguring.
  */
-void cxgbe_dev_stop(struct rte_eth_dev *eth_dev)
+int cxgbe_dev_stop(struct rte_eth_dev *eth_dev)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 
 	CXGBE_FUNC_TRACE();
 
 	if (!(adapter->flags & FULL_INIT_DONE))
-		return;
+		return 0;
 
 	cxgbe_down(pi);
 
@@ -360,47 +436,36 @@ void cxgbe_dev_stop(struct rte_eth_dev *eth_dev)
 	 *  have been disabled
 	 */
 	t4_sge_eth_clear_queues(pi);
+	eth_dev->data->scattered_rx = 0;
+
+	return 0;
 }
 
 int cxgbe_dev_configure(struct rte_eth_dev *eth_dev)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
-	uint64_t unsupported_offloads, configured_offloads;
 	int err;
 
 	CXGBE_FUNC_TRACE();
-	configured_offloads = eth_dev->data->dev_conf.rxmode.offloads;
-	if (!(configured_offloads & DEV_RX_OFFLOAD_CRC_STRIP)) {
-		dev_info(adapter, "can't disable hw crc strip\n");
-		configured_offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
-	}
 
-	unsupported_offloads = configured_offloads & ~CXGBE_RX_OFFLOADS;
-	if (unsupported_offloads) {
-		dev_err(adapter, "Rx offloads 0x%" PRIx64 " are not supported. "
-			"Supported:0x%" PRIx64 "\n",
-			unsupported_offloads, (uint64_t)CXGBE_RX_OFFLOADS);
-		return -ENOTSUP;
-	}
-
-	configured_offloads = eth_dev->data->dev_conf.txmode.offloads;
-	unsupported_offloads = configured_offloads & ~CXGBE_TX_OFFLOADS;
-	if (unsupported_offloads) {
-		dev_err(adapter, "Tx offloads 0x%" PRIx64 " are not supported. "
-			"Supported:0x%" PRIx64 "\n",
-			unsupported_offloads, (uint64_t)CXGBE_TX_OFFLOADS);
-		return -ENOTSUP;
-	}
+	if (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
+		eth_dev->data->dev_conf.rxmode.offloads |=
+			DEV_RX_OFFLOAD_RSS_HASH;
 
 	if (!(adapter->flags & FW_QUEUE_BOUND)) {
-		err = setup_sge_fwevtq(adapter);
+		err = cxgbe_setup_sge_fwevtq(adapter);
 		if (err)
 			return err;
 		adapter->flags |= FW_QUEUE_BOUND;
+		if (is_pf4(adapter)) {
+			err = cxgbe_setup_sge_ctrl_txq(adapter);
+			if (err)
+				return err;
+		}
 	}
 
-	err = cfg_queue_count(eth_dev);
+	err = cxgbe_cfg_queue_count(eth_dev);
 	if (err)
 		return err;
 
@@ -440,31 +505,23 @@ int cxgbe_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 int cxgbe_dev_tx_queue_setup(struct rte_eth_dev *eth_dev,
 			     uint16_t queue_idx, uint16_t nb_desc,
 			     unsigned int socket_id,
-			     const struct rte_eth_txconf *tx_conf)
+			     const struct rte_eth_txconf *tx_conf __rte_unused)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 	struct sge *s = &adapter->sge;
-	struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset + queue_idx];
-	int err = 0;
 	unsigned int temp_nb_desc;
-	uint64_t unsupported_offloads;
-
-	unsupported_offloads = tx_conf->offloads & ~CXGBE_TX_OFFLOADS;
-	if (unsupported_offloads) {
-		dev_err(adapter, "Tx offloads 0x%" PRIx64 " are not supported. "
-			"Supported:0x%" PRIx64 "\n",
-			unsupported_offloads, (uint64_t)CXGBE_TX_OFFLOADS);
-		return -ENOTSUP;
-	}
+	struct sge_eth_txq *txq;
+	int err = 0;
 
+	txq = &s->ethtxq[pi->first_txqset + queue_idx];
 	dev_debug(adapter, "%s: eth_dev->data->nb_tx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; pi->first_qset = %u\n",
 		  __func__, eth_dev->data->nb_tx_queues, queue_idx, nb_desc,
-		  socket_id, pi->first_qset);
+		  socket_id, pi->first_txqset);
 
 	/*  Free up the existing queue  */
 	if (eth_dev->data->tx_queues[queue_idx]) {
-		cxgbe_dev_tx_queue_release(eth_dev->data->tx_queues[queue_idx]);
+		cxgbe_dev_tx_queue_release(eth_dev, queue_idx);
 		eth_dev->data->tx_queues[queue_idx] = NULL;
 	}
 
@@ -497,9 +554,9 @@ int cxgbe_dev_tx_queue_setup(struct rte_eth_dev *eth_dev,
 	return err;
 }
 
-void cxgbe_dev_tx_queue_release(void *q)
+void cxgbe_dev_tx_queue_release(struct rte_eth_dev *eth_dev, uint16_t qid)
 {
-	struct sge_eth_txq *txq = (struct sge_eth_txq *)q;
+	struct sge_eth_txq *txq = eth_dev->data->tx_queues[qid];
 
 	if (txq) {
 		struct port_info *pi = (struct port_info *)
@@ -515,17 +572,16 @@ void cxgbe_dev_tx_queue_release(void *q)
 
 int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 {
-	int ret;
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adap = pi->adapter;
-	struct sge_rspq *q;
+	struct sge_eth_rxq *rxq;
+	int ret;
 
 	dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
 		  __func__, pi->port_id, rx_queue_id);
 
-	q = eth_dev->data->rx_queues[rx_queue_id];
-
-	ret = t4_sge_eth_rxq_start(adap, q);
+	rxq = eth_dev->data->rx_queues[rx_queue_id];
+	ret = t4_sge_eth_rxq_start(adap, rxq);
 	if (ret == 0)
 		eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 
@@ -534,16 +590,16 @@ int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 
 int cxgbe_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 {
-	int ret;
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adap = pi->adapter;
-	struct sge_rspq *q;
+	struct sge_eth_rxq *rxq;
+	int ret;
 
 	dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
 		  __func__, pi->port_id, rx_queue_id);
 
-	q = eth_dev->data->rx_queues[rx_queue_id];
-	ret = t4_sge_eth_rxq_stop(adap, q);
+	rxq = eth_dev->data->rx_queues[rx_queue_id];
+	ret = t4_sge_eth_rxq_stop(adap, rxq);
 	if (ret == 0)
 		eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 
@@ -553,41 +609,32 @@ int cxgbe_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 			     uint16_t queue_idx, uint16_t nb_desc,
 			     unsigned int socket_id,
-			     const struct rte_eth_rxconf *rx_conf,
+			     const struct rte_eth_rxconf *rx_conf __rte_unused,
 			     struct rte_mempool *mp)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	unsigned int pkt_len = eth_dev->data->mtu + RTE_ETHER_HDR_LEN +
+		RTE_ETHER_CRC_LEN;
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
+	struct rte_eth_dev_info dev_info;
 	struct sge *s = &adapter->sge;
-	struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset + queue_idx];
-	int err = 0;
-	int msi_idx = 0;
 	unsigned int temp_nb_desc;
-	struct rte_eth_dev_info dev_info;
-	unsigned int pkt_len = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
-	uint64_t unsupported_offloads, configured_offloads;
-
-	configured_offloads = rx_conf->offloads;
-	if (!(configured_offloads & DEV_RX_OFFLOAD_CRC_STRIP)) {
-		dev_info(adapter, "can't disable hw crc strip\n");
-		configured_offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
-	}
-
-	unsupported_offloads = configured_offloads & ~CXGBE_RX_OFFLOADS;
-	if (unsupported_offloads) {
-		dev_err(adapter, "Rx offloads 0x%" PRIx64 " are not supported. "
-			"Supported:0x%" PRIx64 "\n",
-			unsupported_offloads, (uint64_t)CXGBE_RX_OFFLOADS);
-		return -ENOTSUP;
-	}
+	int err = 0, msi_idx = 0;
+	struct sge_eth_rxq *rxq;
 
+	rxq = &s->ethrxq[pi->first_rxqset + queue_idx];
 	dev_debug(adapter, "%s: eth_dev->data->nb_rx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; mp = %p\n",
 		  __func__, eth_dev->data->nb_rx_queues, queue_idx, nb_desc,
 		  socket_id, mp);
 
-	cxgbe_dev_info_get(eth_dev, &dev_info);
+	err = cxgbe_dev_info_get(eth_dev, &dev_info);
+	if (err != 0) {
+		dev_err(adap, "%s: error during getting ethernet device info",
+			__func__);
+		return err;
+	}
 
-	/* Must accommodate at least ETHER_MIN_MTU */
+	/* Must accommodate at least RTE_ETHER_MIN_MTU */
 	if ((pkt_len < dev_info.min_rx_bufsize) ||
 	    (pkt_len > dev_info.max_rx_pktlen)) {
 		dev_err(adap, "%s: max pkt len must be > %d and <= %d\n",
@@ -598,7 +645,7 @@ int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 
 	/*  Free up the existing queue  */
 	if (eth_dev->data->rx_queues[queue_idx]) {
-		cxgbe_dev_rx_queue_release(eth_dev->data->rx_queues[queue_idx]);
+		cxgbe_dev_rx_queue_release(eth_dev, queue_idx);
 		eth_dev->data->rx_queues[queue_idx] = NULL;
 	}
 
@@ -622,11 +669,10 @@ int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 	}
 
 	rxq->rspq.size = temp_nb_desc;
-	if ((&rxq->fl) != NULL)
-		rxq->fl.size = temp_nb_desc;
+	rxq->fl.size = temp_nb_desc;
 
 	/* Set to jumbo mode if necessary */
-	if (pkt_len > ETHER_MAX_LEN)
+	if (eth_dev->data->mtu > RTE_ETHER_MTU)
 		eth_dev->data->dev_conf.rxmode.offloads |=
 			DEV_RX_OFFLOAD_JUMBO_FRAME;
 	else
@@ -634,7 +680,7 @@ int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 			~DEV_RX_OFFLOAD_JUMBO_FRAME;
 
 	err = t4_sge_alloc_rxq(adapter, &rxq->rspq, false, eth_dev, msi_idx,
-			       &rxq->fl, t4_ethrx_handler,
+			       &rxq->fl, NULL,
 			       is_pf4(adapter) ?
 			       t4_get_tp_ch_map(adapter, pi->tx_chan) : 0, mp,
 			       queue_idx, socket_id);
@@ -645,14 +691,13 @@ int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 	return err;
 }
 
-void cxgbe_dev_rx_queue_release(void *q)
+void cxgbe_dev_rx_queue_release(struct rte_eth_dev *eth_dev, uint16_t qid)
 {
-	struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)q;
-	struct sge_rspq *rq = &rxq->rspq;
+	struct sge_eth_rxq *rxq = eth_dev->data->rx_queues[qid];
 
-	if (rq) {
+	if (rxq) {
 		struct port_info *pi = (struct port_info *)
-				       (rq->eth_dev->data->dev_private);
+				       (rxq->rspq.eth_dev->data->dev_private);
 		struct adapter *adap = pi->adapter;
 
 		dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
@@ -668,7 +713,7 @@ void cxgbe_dev_rx_queue_release(void *q)
 static int cxgbe_dev_stats_get(struct rte_eth_dev *eth_dev,
 				struct rte_eth_stats *eth_stats)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 	struct sge *s = &adapter->sge;
 	struct port_stats ps;
@@ -691,64 +736,367 @@ static int cxgbe_dev_stats_get(struct rte_eth_dev *eth_dev,
 	eth_stats->oerrors  = ps.tx_error_frames;
 
 	for (i = 0; i < pi->n_rx_qsets; i++) {
-		struct sge_eth_rxq *rxq =
-			&s->ethrxq[pi->first_qset + i];
+		struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_rxqset + i];
 
-		eth_stats->q_ipackets[i] = rxq->stats.pkts;
-		eth_stats->q_ibytes[i] = rxq->stats.rx_bytes;
-		eth_stats->ipackets += eth_stats->q_ipackets[i];
-		eth_stats->ibytes += eth_stats->q_ibytes[i];
+		eth_stats->ipackets += rxq->stats.pkts;
+		eth_stats->ibytes += rxq->stats.rx_bytes;
 	}
 
-	for (i = 0; i < pi->n_tx_qsets; i++) {
-		struct sge_eth_txq *txq =
-			&s->ethtxq[pi->first_qset + i];
-
-		eth_stats->q_opackets[i] = txq->stats.pkts;
-		eth_stats->q_obytes[i] = txq->stats.tx_bytes;
-		eth_stats->q_errors[i] = txq->stats.mapping_err;
-	}
 	return 0;
 }
 
 /*
  * Reset port statistics.
  */
-static void cxgbe_dev_stats_reset(struct rte_eth_dev *eth_dev)
+static int cxgbe_dev_stats_reset(struct rte_eth_dev *eth_dev)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 	struct sge *s = &adapter->sge;
 	unsigned int i;
 
 	cxgbe_stats_reset(pi);
 	for (i = 0; i < pi->n_rx_qsets; i++) {
-		struct sge_eth_rxq *rxq =
-			&s->ethrxq[pi->first_qset + i];
+		struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_rxqset + i];
 
-		rxq->stats.pkts = 0;
-		rxq->stats.rx_bytes = 0;
+		memset(&rxq->stats, 0, sizeof(rxq->stats));
 	}
 	for (i = 0; i < pi->n_tx_qsets; i++) {
-		struct sge_eth_txq *txq =
-			&s->ethtxq[pi->first_qset + i];
+		struct sge_eth_txq *txq = &s->ethtxq[pi->first_txqset + i];
+
+		memset(&txq->stats, 0, sizeof(txq->stats));
+	}
+
+	return 0;
+}
+
+/* Store extended statistics names and its offset in stats structure  */
+struct cxgbe_dev_xstats_name_off {
+	char name[RTE_ETH_XSTATS_NAME_SIZE];
+	unsigned int offset;
+};
+
+static const struct cxgbe_dev_xstats_name_off cxgbe_dev_rxq_stats_strings[] = {
+	{"packets", offsetof(struct sge_eth_rx_stats, pkts)},
+	{"bytes", offsetof(struct sge_eth_rx_stats, rx_bytes)},
+	{"checksum_offloads", offsetof(struct sge_eth_rx_stats, rx_cso)},
+	{"vlan_extractions", offsetof(struct sge_eth_rx_stats, vlan_ex)},
+	{"dropped_packets", offsetof(struct sge_eth_rx_stats, rx_drops)},
+};
+
+static const struct cxgbe_dev_xstats_name_off cxgbe_dev_txq_stats_strings[] = {
+	{"packets", offsetof(struct sge_eth_tx_stats, pkts)},
+	{"bytes", offsetof(struct sge_eth_tx_stats, tx_bytes)},
+	{"tso_requests", offsetof(struct sge_eth_tx_stats, tso)},
+	{"checksum_offloads", offsetof(struct sge_eth_tx_stats, tx_cso)},
+	{"vlan_insertions", offsetof(struct sge_eth_tx_stats, vlan_ins)},
+	{"packet_mapping_errors",
+	 offsetof(struct sge_eth_tx_stats, mapping_err)},
+	{"coalesced_wrs", offsetof(struct sge_eth_tx_stats, coal_wr)},
+	{"coalesced_packets", offsetof(struct sge_eth_tx_stats, coal_pkts)},
+};
+
+static const struct cxgbe_dev_xstats_name_off cxgbe_dev_port_stats_strings[] = {
+	{"tx_bytes", offsetof(struct port_stats, tx_octets)},
+	{"tx_packets", offsetof(struct port_stats, tx_frames)},
+	{"tx_broadcast_packets", offsetof(struct port_stats, tx_bcast_frames)},
+	{"tx_multicast_packets", offsetof(struct port_stats, tx_mcast_frames)},
+	{"tx_unicast_packets", offsetof(struct port_stats, tx_ucast_frames)},
+	{"tx_error_packets", offsetof(struct port_stats, tx_error_frames)},
+	{"tx_size_64_packets", offsetof(struct port_stats, tx_frames_64)},
+	{"tx_size_65_to_127_packets",
+	 offsetof(struct port_stats, tx_frames_65_127)},
+	{"tx_size_128_to_255_packets",
+	 offsetof(struct port_stats, tx_frames_128_255)},
+	{"tx_size_256_to_511_packets",
+	 offsetof(struct port_stats, tx_frames_256_511)},
+	{"tx_size_512_to_1023_packets",
+	 offsetof(struct port_stats, tx_frames_512_1023)},
+	{"tx_size_1024_to_1518_packets",
+	 offsetof(struct port_stats, tx_frames_1024_1518)},
+	{"tx_size_1519_to_max_packets",
+	 offsetof(struct port_stats, tx_frames_1519_max)},
+	{"tx_drop_packets", offsetof(struct port_stats, tx_drop)},
+	{"tx_pause_frames", offsetof(struct port_stats, tx_pause)},
+	{"tx_ppp_pri0_packets", offsetof(struct port_stats, tx_ppp0)},
+	{"tx_ppp_pri1_packets", offsetof(struct port_stats, tx_ppp1)},
+	{"tx_ppp_pri2_packets", offsetof(struct port_stats, tx_ppp2)},
+	{"tx_ppp_pri3_packets", offsetof(struct port_stats, tx_ppp3)},
+	{"tx_ppp_pri4_packets", offsetof(struct port_stats, tx_ppp4)},
+	{"tx_ppp_pri5_packets", offsetof(struct port_stats, tx_ppp5)},
+	{"tx_ppp_pri6_packets", offsetof(struct port_stats, tx_ppp6)},
+	{"tx_ppp_pri7_packets", offsetof(struct port_stats, tx_ppp7)},
+	{"rx_bytes", offsetof(struct port_stats, rx_octets)},
+	{"rx_packets", offsetof(struct port_stats, rx_frames)},
+	{"rx_broadcast_packets", offsetof(struct port_stats, rx_bcast_frames)},
+	{"rx_multicast_packets", offsetof(struct port_stats, rx_mcast_frames)},
+	{"rx_unicast_packets", offsetof(struct port_stats, rx_ucast_frames)},
+	{"rx_too_long_packets", offsetof(struct port_stats, rx_too_long)},
+	{"rx_jabber_packets", offsetof(struct port_stats, rx_jabber)},
+	{"rx_fcs_error_packets", offsetof(struct port_stats, rx_fcs_err)},
+	{"rx_length_error_packets", offsetof(struct port_stats, rx_len_err)},
+	{"rx_symbol_error_packets",
+	 offsetof(struct port_stats, rx_symbol_err)},
+	{"rx_short_packets", offsetof(struct port_stats, rx_runt)},
+	{"rx_size_64_packets", offsetof(struct port_stats, rx_frames_64)},
+	{"rx_size_65_to_127_packets",
+	 offsetof(struct port_stats, rx_frames_65_127)},
+	{"rx_size_128_to_255_packets",
+	 offsetof(struct port_stats, rx_frames_128_255)},
+	{"rx_size_256_to_511_packets",
+	 offsetof(struct port_stats, rx_frames_256_511)},
+	{"rx_size_512_to_1023_packets",
+	 offsetof(struct port_stats, rx_frames_512_1023)},
+	{"rx_size_1024_to_1518_packets",
+	 offsetof(struct port_stats, rx_frames_1024_1518)},
+	{"rx_size_1519_to_max_packets",
+	 offsetof(struct port_stats, rx_frames_1519_max)},
+	{"rx_pause_packets", offsetof(struct port_stats, rx_pause)},
+	{"rx_ppp_pri0_packets", offsetof(struct port_stats, rx_ppp0)},
+	{"rx_ppp_pri1_packets", offsetof(struct port_stats, rx_ppp1)},
+	{"rx_ppp_pri2_packets", offsetof(struct port_stats, rx_ppp2)},
+	{"rx_ppp_pri3_packets", offsetof(struct port_stats, rx_ppp3)},
+	{"rx_ppp_pri4_packets", offsetof(struct port_stats, rx_ppp4)},
+	{"rx_ppp_pri5_packets", offsetof(struct port_stats, rx_ppp5)},
+	{"rx_ppp_pri6_packets", offsetof(struct port_stats, rx_ppp6)},
+	{"rx_ppp_pri7_packets", offsetof(struct port_stats, rx_ppp7)},
+	{"rx_bg0_dropped_packets", offsetof(struct port_stats, rx_ovflow0)},
+	{"rx_bg1_dropped_packets", offsetof(struct port_stats, rx_ovflow1)},
+	{"rx_bg2_dropped_packets", offsetof(struct port_stats, rx_ovflow2)},
+	{"rx_bg3_dropped_packets", offsetof(struct port_stats, rx_ovflow3)},
+	{"rx_bg0_truncated_packets", offsetof(struct port_stats, rx_trunc0)},
+	{"rx_bg1_truncated_packets", offsetof(struct port_stats, rx_trunc1)},
+	{"rx_bg2_truncated_packets", offsetof(struct port_stats, rx_trunc2)},
+	{"rx_bg3_truncated_packets", offsetof(struct port_stats, rx_trunc3)},
+};
+
+static const struct cxgbe_dev_xstats_name_off
+cxgbevf_dev_port_stats_strings[] = {
+	{"tx_bytes", offsetof(struct port_stats, tx_octets)},
+	{"tx_broadcast_packets", offsetof(struct port_stats, tx_bcast_frames)},
+	{"tx_multicast_packets", offsetof(struct port_stats, tx_mcast_frames)},
+	{"tx_unicast_packets", offsetof(struct port_stats, tx_ucast_frames)},
+	{"tx_drop_packets", offsetof(struct port_stats, tx_drop)},
+	{"rx_broadcast_packets", offsetof(struct port_stats, rx_bcast_frames)},
+	{"rx_multicast_packets", offsetof(struct port_stats, rx_mcast_frames)},
+	{"rx_unicast_packets", offsetof(struct port_stats, rx_ucast_frames)},
+	{"rx_length_error_packets", offsetof(struct port_stats, rx_len_err)},
+};
+
+#define CXGBE_NB_RXQ_STATS RTE_DIM(cxgbe_dev_rxq_stats_strings)
+#define CXGBE_NB_TXQ_STATS RTE_DIM(cxgbe_dev_txq_stats_strings)
+#define CXGBE_NB_PORT_STATS RTE_DIM(cxgbe_dev_port_stats_strings)
+#define CXGBEVF_NB_PORT_STATS RTE_DIM(cxgbevf_dev_port_stats_strings)
+
+static u16 cxgbe_dev_xstats_count(struct port_info *pi)
+{
+	u16 count;
+
+	count = (pi->n_tx_qsets * CXGBE_NB_TXQ_STATS) +
+		(pi->n_rx_qsets * CXGBE_NB_RXQ_STATS);
+
+	if (is_pf4(pi->adapter) != 0)
+		count += CXGBE_NB_PORT_STATS;
+	else
+		count += CXGBEVF_NB_PORT_STATS;
+
+	return count;
+}
+
+static int cxgbe_dev_xstats(struct rte_eth_dev *dev,
+			    struct rte_eth_xstat_name *xstats_names,
+			    struct rte_eth_xstat *xstats, unsigned int size)
+{
+	const struct cxgbe_dev_xstats_name_off *xstats_str;
+	struct port_info *pi = dev->data->dev_private;
+	struct adapter *adap = pi->adapter;
+	struct sge *s = &adap->sge;
+	u16 count, i, qid, nstats;
+	struct port_stats ps;
+	u64 *stats_ptr;
+
+	count = cxgbe_dev_xstats_count(pi);
+	if (size < count)
+		return count;
+
+	if (is_pf4(adap) != 0) {
+		/* port stats for PF*/
+		cxgbe_stats_get(pi, &ps);
+		xstats_str = cxgbe_dev_port_stats_strings;
+		nstats = CXGBE_NB_PORT_STATS;
+	} else {
+		/* port stats for VF*/
+		cxgbevf_stats_get(pi, &ps);
+		xstats_str = cxgbevf_dev_port_stats_strings;
+		nstats = CXGBEVF_NB_PORT_STATS;
+	}
+
+	count = 0;
+	for (i = 0; i < nstats; i++, count++) {
+		if (xstats_names != NULL)
+			snprintf(xstats_names[count].name,
+				 sizeof(xstats_names[count].name),
+				 "%s", xstats_str[i].name);
+		if (xstats != NULL) {
+			stats_ptr = RTE_PTR_ADD(&ps,
+						xstats_str[i].offset);
+			xstats[count].value = *stats_ptr;
+			xstats[count].id = count;
+		}
+	}
+
+	/* per-txq stats */
+	xstats_str = cxgbe_dev_txq_stats_strings;
+	for (qid = 0; qid < pi->n_tx_qsets; qid++) {
+		struct sge_eth_txq *txq = &s->ethtxq[pi->first_txqset + qid];
+
+		for (i = 0; i < CXGBE_NB_TXQ_STATS; i++, count++) {
+			if (xstats_names != NULL)
+				snprintf(xstats_names[count].name,
+					 sizeof(xstats_names[count].name),
+					 "tx_q%u_%s",
+					 qid, xstats_str[i].name);
+			if (xstats != NULL) {
+				stats_ptr = RTE_PTR_ADD(&txq->stats,
+							xstats_str[i].offset);
+				xstats[count].value = *stats_ptr;
+				xstats[count].id = count;
+			}
+		}
+	}
+
+	/* per-rxq stats */
+	xstats_str = cxgbe_dev_rxq_stats_strings;
+	for (qid = 0; qid < pi->n_rx_qsets; qid++) {
+		struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_rxqset + qid];
+
+		for (i = 0; i < CXGBE_NB_RXQ_STATS; i++, count++) {
+			if (xstats_names != NULL)
+				snprintf(xstats_names[count].name,
+					 sizeof(xstats_names[count].name),
+					 "rx_q%u_%s",
+					 qid, xstats_str[i].name);
+			if (xstats != NULL) {
+				stats_ptr = RTE_PTR_ADD(&rxq->stats,
+							xstats_str[i].offset);
+				xstats[count].value = *stats_ptr;
+				xstats[count].id = count;
+			}
+		}
+	}
+
+	return count;
+}
+
+/* Get port extended statistics by ID. */
+int cxgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev,
+			       const uint64_t *ids, uint64_t *values,
+			       unsigned int n)
+{
+	struct port_info *pi = dev->data->dev_private;
+	struct rte_eth_xstat *xstats_copy;
+	u16 count, i;
+	int ret = 0;
+
+	count = cxgbe_dev_xstats_count(pi);
+	if (ids == NULL || values == NULL)
+		return count;
+
+	xstats_copy = rte_calloc(NULL, count, sizeof(*xstats_copy), 0);
+	if (xstats_copy == NULL)
+		return -ENOMEM;
+
+	cxgbe_dev_xstats(dev, NULL, xstats_copy, count);
+
+	for (i = 0; i < n; i++) {
+		if (ids[i] >= count) {
+			ret = -EINVAL;
+			goto out_err;
+		}
+		values[i] = xstats_copy[ids[i]].value;
+	}
+
+	ret = n;
+
+out_err:
+	rte_free(xstats_copy);
+	return ret;
+}
+
+/* Get names of port extended statistics by ID. */
+int cxgbe_dev_xstats_get_names_by_id(struct rte_eth_dev *dev,
+					    const uint64_t *ids,
+					    struct rte_eth_xstat_name *xnames,
+					    unsigned int n)
+{
+	struct port_info *pi = dev->data->dev_private;
+	struct rte_eth_xstat_name *xnames_copy;
+	u16 count, i;
+	int ret = 0;
+
+	count = cxgbe_dev_xstats_count(pi);
+	if (ids == NULL || xnames == NULL)
+		return count;
+
+	xnames_copy = rte_calloc(NULL, count, sizeof(*xnames_copy), 0);
+	if (xnames_copy == NULL)
+		return -ENOMEM;
+
+	cxgbe_dev_xstats(dev, xnames_copy, NULL, count);
 
-		txq->stats.pkts = 0;
-		txq->stats.tx_bytes = 0;
-		txq->stats.mapping_err = 0;
+	for (i = 0; i < n; i++) {
+		if (ids[i] >= count) {
+			ret = -EINVAL;
+			goto out_err;
+		}
+		rte_strlcpy(xnames[i].name, xnames_copy[ids[i]].name,
+			    sizeof(xnames[i].name));
 	}
+
+	ret = n;
+
+out_err:
+	rte_free(xnames_copy);
+	return ret;
+}
+
+/* Get port extended statistics. */
+int cxgbe_dev_xstats_get(struct rte_eth_dev *dev,
+			 struct rte_eth_xstat *xstats, unsigned int n)
+{
+	return cxgbe_dev_xstats(dev, NULL, xstats, n);
+}
+
+/* Get names of port extended statistics. */
+int cxgbe_dev_xstats_get_names(struct rte_eth_dev *dev,
+			       struct rte_eth_xstat_name *xstats_names,
+			       unsigned int n)
+{
+	return cxgbe_dev_xstats(dev, xstats_names, NULL, n);
+}
+
+/* Reset port extended statistics. */
+static int cxgbe_dev_xstats_reset(struct rte_eth_dev *dev)
+{
+	return cxgbe_dev_stats_reset(dev);
 }
 
 static int cxgbe_flow_ctrl_get(struct rte_eth_dev *eth_dev,
 			       struct rte_eth_fc_conf *fc_conf)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct link_config *lc = &pi->link_cfg;
-	int rx_pause, tx_pause;
+	u8 rx_pause = 0, tx_pause = 0;
+	u32 caps = lc->link_caps;
+
+	if (caps & FW_PORT_CAP32_ANEG)
+		fc_conf->autoneg = 1;
 
-	fc_conf->autoneg = lc->fc & PAUSE_AUTONEG;
-	rx_pause = lc->fc & PAUSE_RX;
-	tx_pause = lc->fc & PAUSE_TX;
+	if (caps & FW_PORT_CAP32_FC_TX)
+		tx_pause = 1;
+
+	if (caps & FW_PORT_CAP32_FC_RX)
+		rx_pause = 1;
 
 	if (rx_pause && tx_pause)
 		fc_conf->mode = RTE_FC_FULL;
@@ -764,31 +1112,40 @@ static int cxgbe_flow_ctrl_get(struct rte_eth_dev *eth_dev,
 static int cxgbe_flow_ctrl_set(struct rte_eth_dev *eth_dev,
 			       struct rte_eth_fc_conf *fc_conf)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
-	struct adapter *adapter = pi->adapter;
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct link_config *lc = &pi->link_cfg;
+	u32 new_caps = lc->admin_caps;
+	u8 tx_pause = 0, rx_pause = 0;
+	int ret;
 
-	if (lc->pcaps & FW_PORT_CAP32_ANEG) {
-		if (fc_conf->autoneg)
-			lc->requested_fc |= PAUSE_AUTONEG;
-		else
-			lc->requested_fc &= ~PAUSE_AUTONEG;
+	if (fc_conf->mode == RTE_FC_FULL) {
+		tx_pause = 1;
+		rx_pause = 1;
+	} else if (fc_conf->mode == RTE_FC_TX_PAUSE) {
+		tx_pause = 1;
+	} else if (fc_conf->mode == RTE_FC_RX_PAUSE) {
+		rx_pause = 1;
 	}
 
-	if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
-	    (fc_conf->mode & RTE_FC_RX_PAUSE))
-		lc->requested_fc |= PAUSE_RX;
-	else
-		lc->requested_fc &= ~PAUSE_RX;
+	ret = t4_set_link_pause(pi, fc_conf->autoneg, tx_pause,
+				rx_pause, &new_caps);
+	if (ret != 0)
+		return ret;
 
-	if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
-	    (fc_conf->mode & RTE_FC_TX_PAUSE))
-		lc->requested_fc |= PAUSE_TX;
-	else
-		lc->requested_fc &= ~PAUSE_TX;
+	if (!fc_conf->autoneg) {
+		if (lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE)
+			new_caps |= FW_PORT_CAP32_FORCE_PAUSE;
+	} else {
+		new_caps &= ~FW_PORT_CAP32_FORCE_PAUSE;
+	}
 
-	return t4_link_l1cfg(adapter, adapter->mbox, pi->tx_chan,
-			     &pi->link_cfg);
+	if (new_caps != lc->admin_caps) {
+		ret = t4_link_l1cfg(pi, new_caps);
+		if (ret == 0)
+			lc->admin_caps = new_caps;
+	}
+
+	return ret;
 }
 
 const uint32_t *
@@ -810,7 +1167,7 @@ cxgbe_dev_supported_ptypes_get(struct rte_eth_dev *eth_dev)
 static int cxgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
 				     struct rte_eth_rss_conf *rss_conf)
 {
-	struct port_info *pi = (struct port_info *)(dev->data->dev_private);
+	struct port_info *pi = dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 	int err;
 
@@ -840,7 +1197,7 @@ static int cxgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
 static int cxgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 				       struct rte_eth_rss_conf *rss_conf)
 {
-	struct port_info *pi = (struct port_info *)(dev->data->dev_private);
+	struct port_info *pi = dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 	u64 rss_hf = 0;
 	u64 flags = 0;
@@ -853,13 +1210,13 @@ static int cxgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 		return err;
 
 	if (flags & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) {
-		rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
+		rss_hf |= CXGBE_RSS_HF_TCP_IPV6_MASK;
 		if (flags & F_FW_RSS_VI_CONFIG_CMD_UDPEN)
-			rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
+			rss_hf |= CXGBE_RSS_HF_UDP_IPV6_MASK;
 	}
 
 	if (flags & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
-		rss_hf |= ETH_RSS_IPV6;
+		rss_hf |= CXGBE_RSS_HF_IPV6_MASK;
 
 	if (flags & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) {
 		rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
@@ -868,7 +1225,7 @@ static int cxgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 	}
 
 	if (flags & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
-		rss_hf |= ETH_RSS_IPV4;
+		rss_hf |= CXGBE_RSS_HF_IPV4_MASK;
 
 	rss_conf->rss_hf = rss_hf;
 
@@ -887,6 +1244,69 @@ static int cxgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static int cxgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
+				     struct rte_eth_rss_reta_entry64 *reta_conf,
+				     uint16_t reta_size)
+{
+	struct port_info *pi = dev->data->dev_private;
+	struct adapter *adapter = pi->adapter;
+	u16 i, idx, shift, *rss;
+	int ret;
+
+	if (!(adapter->flags & FULL_INIT_DONE))
+		return -ENOMEM;
+
+	if (!reta_size || reta_size > pi->rss_size)
+		return -EINVAL;
+
+	rss = rte_calloc(NULL, pi->rss_size, sizeof(u16), 0);
+	if (!rss)
+		return -ENOMEM;
+
+	rte_memcpy(rss, pi->rss, pi->rss_size * sizeof(u16));
+	for (i = 0; i < reta_size; i++) {
+		idx = i / RTE_RETA_GROUP_SIZE;
+		shift = i % RTE_RETA_GROUP_SIZE;
+		if (!(reta_conf[idx].mask & (1ULL << shift)))
+			continue;
+
+		rss[i] = reta_conf[idx].reta[shift];
+	}
+
+	ret = cxgbe_write_rss(pi, rss);
+	if (!ret)
+		rte_memcpy(pi->rss, rss, pi->rss_size * sizeof(u16));
+
+	rte_free(rss);
+	return ret;
+}
+
+static int cxgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
+				    struct rte_eth_rss_reta_entry64 *reta_conf,
+				    uint16_t reta_size)
+{
+	struct port_info *pi = dev->data->dev_private;
+	struct adapter *adapter = pi->adapter;
+	u16 i, idx, shift;
+
+	if (!(adapter->flags & FULL_INIT_DONE))
+		return -ENOMEM;
+
+	if (!reta_size || reta_size > pi->rss_size)
+		return -EINVAL;
+
+	for (i = 0; i < reta_size; i++) {
+		idx = i / RTE_RETA_GROUP_SIZE;
+		shift = i % RTE_RETA_GROUP_SIZE;
+		if (!(reta_conf[idx].mask & (1ULL << shift)))
+			continue;
+
+		reta_conf[idx].reta[shift] = pi->rss[i];
+	}
+
+	return 0;
+}
+
 static int cxgbe_get_eeprom_length(struct rte_eth_dev *dev)
 {
 	RTE_SET_USED(dev);
@@ -949,7 +1369,7 @@ static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
 static int cxgbe_get_eeprom(struct rte_eth_dev *dev,
 			    struct rte_dev_eeprom_info *e)
 {
-	struct port_info *pi = (struct port_info *)(dev->data->dev_private);
+	struct port_info *pi = dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 	u32 i, err = 0;
 	u8 *buf = rte_zmalloc(NULL, EEPROMSIZE, 0);
@@ -970,7 +1390,7 @@ static int cxgbe_get_eeprom(struct rte_eth_dev *dev,
 static int cxgbe_set_eeprom(struct rte_eth_dev *dev,
 			    struct rte_dev_eeprom_info *eeprom)
 {
-	struct port_info *pi = (struct port_info *)(dev->data->dev_private);
+	struct port_info *pi = dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 	u8 *buf;
 	int err = 0;
@@ -1028,7 +1448,7 @@ out:
 
 static int cxgbe_get_regs_len(struct rte_eth_dev *eth_dev)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 
 	return t4_get_regs_len(adapter) / sizeof(uint32_t);
@@ -1037,7 +1457,7 @@ static int cxgbe_get_regs_len(struct rte_eth_dev *eth_dev)
 static int cxgbe_get_regs(struct rte_eth_dev *eth_dev,
 			  struct rte_dev_reg_info *regs)
 {
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = pi->adapter;
 
 	regs->version = CHELSIO_CHIP_VERSION(adapter->params.chip) |
@@ -1056,20 +1476,163 @@ static int cxgbe_get_regs(struct rte_eth_dev *eth_dev,
 	return 0;
 }
 
-void cxgbe_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr)
+int cxgbe_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *addr)
 {
-	struct port_info *pi = (struct port_info *)(dev->data->dev_private);
-	struct adapter *adapter = pi->adapter;
+	struct port_info *pi = dev->data->dev_private;
 	int ret;
 
-	ret = t4_change_mac(adapter, adapter->mbox, pi->viid,
-			    pi->xact_addr_filt, (u8 *)addr, true, true);
+	ret = cxgbe_mpstcam_modify(pi, (int)pi->xact_addr_filt, (u8 *)addr);
 	if (ret < 0) {
 		dev_err(adapter, "failed to set mac addr; err = %d\n",
 			ret);
-		return;
+		return ret;
 	}
 	pi->xact_addr_filt = ret;
+	return 0;
+}
+
+static int cxgbe_fec_get_capa_speed_to_fec(struct link_config *lc,
+					   struct rte_eth_fec_capa *capa_arr)
+{
+	int num = 0;
+
+	if (lc->pcaps & FW_PORT_CAP32_SPEED_100G) {
+		if (capa_arr) {
+			capa_arr[num].speed = ETH_SPEED_NUM_100G;
+			capa_arr[num].capa = RTE_ETH_FEC_MODE_CAPA_MASK(NOFEC) |
+					     RTE_ETH_FEC_MODE_CAPA_MASK(RS);
+		}
+		num++;
+	}
+
+	if (lc->pcaps & FW_PORT_CAP32_SPEED_50G) {
+		if (capa_arr) {
+			capa_arr[num].speed = ETH_SPEED_NUM_50G;
+			capa_arr[num].capa = RTE_ETH_FEC_MODE_CAPA_MASK(NOFEC) |
+					     RTE_ETH_FEC_MODE_CAPA_MASK(BASER);
+		}
+		num++;
+	}
+
+	if (lc->pcaps & FW_PORT_CAP32_SPEED_25G) {
+		if (capa_arr) {
+			capa_arr[num].speed = ETH_SPEED_NUM_25G;
+			capa_arr[num].capa = RTE_ETH_FEC_MODE_CAPA_MASK(NOFEC) |
+					     RTE_ETH_FEC_MODE_CAPA_MASK(BASER) |
+					     RTE_ETH_FEC_MODE_CAPA_MASK(RS);
+		}
+		num++;
+	}
+
+	return num;
+}
+
+static int cxgbe_fec_get_capability(struct rte_eth_dev *dev,
+				    struct rte_eth_fec_capa *speed_fec_capa,
+				    unsigned int num)
+{
+	struct port_info *pi = dev->data->dev_private;
+	struct link_config *lc = &pi->link_cfg;
+	u8 num_entries;
+
+	if (!(lc->pcaps & V_FW_PORT_CAP32_FEC(M_FW_PORT_CAP32_FEC)))
+		return -EOPNOTSUPP;
+
+	num_entries = cxgbe_fec_get_capa_speed_to_fec(lc, NULL);
+	if (!speed_fec_capa || num < num_entries)
+		return num_entries;
+
+	return cxgbe_fec_get_capa_speed_to_fec(lc, speed_fec_capa);
+}
+
+static int cxgbe_fec_get(struct rte_eth_dev *dev, uint32_t *fec_capa)
+{
+	struct port_info *pi = dev->data->dev_private;
+	struct link_config *lc = &pi->link_cfg;
+	u32 fec_caps = 0, caps = lc->link_caps;
+
+	if (!(lc->pcaps & V_FW_PORT_CAP32_FEC(M_FW_PORT_CAP32_FEC)))
+		return -EOPNOTSUPP;
+
+	if (caps & FW_PORT_CAP32_FEC_RS)
+		fec_caps = RTE_ETH_FEC_MODE_CAPA_MASK(RS);
+	else if (caps & FW_PORT_CAP32_FEC_BASER_RS)
+		fec_caps = RTE_ETH_FEC_MODE_CAPA_MASK(BASER);
+	else
+		fec_caps = RTE_ETH_FEC_MODE_CAPA_MASK(NOFEC);
+
+	*fec_capa = fec_caps;
+	return 0;
+}
+
+static int cxgbe_fec_set(struct rte_eth_dev *dev, uint32_t fec_capa)
+{
+	struct port_info *pi = dev->data->dev_private;
+	u8 fec_rs = 0, fec_baser = 0, fec_none = 0;
+	struct link_config *lc = &pi->link_cfg;
+	u32 new_caps = lc->admin_caps;
+	int ret;
+
+	if (!(lc->pcaps & V_FW_PORT_CAP32_FEC(M_FW_PORT_CAP32_FEC)))
+		return -EOPNOTSUPP;
+
+	if (!fec_capa)
+		return -EINVAL;
+
+	if (fec_capa & RTE_ETH_FEC_MODE_CAPA_MASK(AUTO))
+		goto set_fec;
+
+	if (fec_capa & RTE_ETH_FEC_MODE_CAPA_MASK(NOFEC))
+		fec_none = 1;
+
+	if (fec_capa & RTE_ETH_FEC_MODE_CAPA_MASK(BASER))
+		fec_baser = 1;
+
+	if (fec_capa & RTE_ETH_FEC_MODE_CAPA_MASK(RS))
+		fec_rs = 1;
+
+set_fec:
+	ret = t4_set_link_fec(pi, fec_rs, fec_baser, fec_none, &new_caps);
+	if (ret != 0)
+		return ret;
+
+	if (lc->pcaps & FW_PORT_CAP32_FORCE_FEC)
+		new_caps |= FW_PORT_CAP32_FORCE_FEC;
+	else
+		new_caps &= ~FW_PORT_CAP32_FORCE_FEC;
+
+	if (new_caps != lc->admin_caps) {
+		ret = t4_link_l1cfg(pi, new_caps);
+		if (ret == 0)
+			lc->admin_caps = new_caps;
+	}
+
+	return ret;
+}
+
+int cxgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
+			 size_t fw_size)
+{
+	struct port_info *pi = dev->data->dev_private;
+	struct adapter *adapter = pi->adapter;
+	int ret;
+
+	if (adapter->params.fw_vers == 0)
+		return -EIO;
+
+	ret = snprintf(fw_version, fw_size, "%u.%u.%u.%u",
+		       G_FW_HDR_FW_VER_MAJOR(adapter->params.fw_vers),
+		       G_FW_HDR_FW_VER_MINOR(adapter->params.fw_vers),
+		       G_FW_HDR_FW_VER_MICRO(adapter->params.fw_vers),
+		       G_FW_HDR_FW_VER_BUILD(adapter->params.fw_vers));
+	if (ret < 0)
+		return -EINVAL;
+
+	ret += 1;
+	if (fw_size < (size_t)ret)
+		return ret;
+
+	return 0;
 }
 
 static const struct eth_dev_ops cxgbe_eth_dev_ops = {
@@ -1084,6 +1647,8 @@ static const struct eth_dev_ops cxgbe_eth_dev_ops = {
 	.dev_infos_get		= cxgbe_dev_info_get,
 	.dev_supported_ptypes_get = cxgbe_dev_supported_ptypes_get,
 	.link_update		= cxgbe_dev_link_update,
+	.dev_set_link_up        = cxgbe_dev_set_link_up,
+	.dev_set_link_down      = cxgbe_dev_set_link_down,
 	.mtu_set		= cxgbe_dev_mtu_set,
 	.tx_queue_setup         = cxgbe_dev_tx_queue_setup,
 	.tx_queue_start		= cxgbe_dev_tx_queue_start,
@@ -1093,8 +1658,14 @@ static const struct eth_dev_ops cxgbe_eth_dev_ops = {
 	.rx_queue_start		= cxgbe_dev_rx_queue_start,
 	.rx_queue_stop		= cxgbe_dev_rx_queue_stop,
 	.rx_queue_release	= cxgbe_dev_rx_queue_release,
+	.flow_ops_get           = cxgbe_dev_flow_ops_get,
 	.stats_get		= cxgbe_dev_stats_get,
 	.stats_reset		= cxgbe_dev_stats_reset,
+	.xstats_get             = cxgbe_dev_xstats_get,
+	.xstats_get_by_id       = cxgbe_dev_xstats_get_by_id,
+	.xstats_get_names       = cxgbe_dev_xstats_get_names,
+	.xstats_get_names_by_id = cxgbe_dev_xstats_get_names_by_id,
+	.xstats_reset           = cxgbe_dev_xstats_reset,
 	.flow_ctrl_get		= cxgbe_flow_ctrl_get,
 	.flow_ctrl_set		= cxgbe_flow_ctrl_set,
 	.get_eeprom_length	= cxgbe_get_eeprom_length,
@@ -1104,6 +1675,12 @@ static const struct eth_dev_ops cxgbe_eth_dev_ops = {
 	.rss_hash_update	= cxgbe_dev_rss_hash_update,
 	.rss_hash_conf_get	= cxgbe_dev_rss_hash_conf_get,
 	.mac_addr_set		= cxgbe_mac_addr_set,
+	.reta_update            = cxgbe_dev_rss_reta_update,
+	.reta_query             = cxgbe_dev_rss_reta_query,
+	.fec_get_capability     = cxgbe_fec_get_capability,
+	.fec_get                = cxgbe_fec_get,
+	.fec_set                = cxgbe_fec_set,
+	.fw_version_get         = cxgbe_fw_version_get,
 };
 
 /*
@@ -1113,7 +1690,7 @@ static const struct eth_dev_ops cxgbe_eth_dev_ops = {
 static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev)
 {
 	struct rte_pci_device *pci_dev;
-	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct port_info *pi = eth_dev->data->dev_private;
 	struct adapter *adapter = NULL;
 	char name[RTE_ETH_NAME_MAX_LEN];
 	int err = 0;
@@ -1146,6 +1723,7 @@ static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev)
 					eth_dev->rx_pkt_burst;
 				rest_eth_dev->tx_pkt_burst =
 					eth_dev->tx_pkt_burst;
+				rte_eth_dev_probing_finish(rest_eth_dev);
 			}
 		}
 		return 0;
@@ -1167,6 +1745,8 @@ static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev)
 	adapter->eth_dev = eth_dev;
 	pi->adapter = adapter;
 
+	cxgbe_process_devargs(adapter);
+
 	err = cxgbe_probe(adapter);
 	if (err) {
 		dev_err(adapter, "%s: cxgbe probe failed with err %d\n",
@@ -1181,6 +1761,19 @@ out_free_adapter:
 	return err;
 }
 
+static int eth_cxgbe_dev_uninit(struct rte_eth_dev *eth_dev)
+{
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+	uint16_t port_id;
+	int err = 0;
+
+	/* Free up other ports and all resources */
+	RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device)
+		err |= rte_eth_dev_close(port_id);
+
+	return err == 0 ? 0 : -EIO;
+}
+
 static int eth_cxgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	struct rte_pci_device *pci_dev)
 {
@@ -1190,7 +1783,7 @@ static int eth_cxgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 
 static int eth_cxgbe_pci_remove(struct rte_pci_device *pci_dev)
 {
-	return rte_eth_dev_pci_generic_remove(pci_dev, NULL);
+	return rte_eth_dev_pci_generic_remove(pci_dev, eth_cxgbe_dev_uninit);
 }
 
 static struct rte_pci_driver rte_cxgbe_pmd = {
@@ -1203,3 +1796,10 @@ static struct rte_pci_driver rte_cxgbe_pmd = {
 RTE_PMD_REGISTER_PCI(net_cxgbe, rte_cxgbe_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_cxgbe, cxgb4_pci_tbl);
 RTE_PMD_REGISTER_KMOD_DEP(net_cxgbe, "* igb_uio | uio_pci_generic | vfio-pci");
+RTE_PMD_REGISTER_PARAM_STRING(net_cxgbe,
+			      CXGBE_DEVARG_CMN_KEEP_OVLAN "=<0|1> "
+			      CXGBE_DEVARG_CMN_TX_MODE_LATENCY "=<0|1> "
+			      CXGBE_DEVARG_PF_FILTER_MODE "=<uint32> "
+			      CXGBE_DEVARG_PF_FILTER_MASK "=<uint32> ");
+RTE_LOG_REGISTER_DEFAULT(cxgbe_logtype, NOTICE);
+RTE_LOG_REGISTER_SUFFIX(cxgbe_mbox_logtype, mbox, NOTICE);