X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fliquidio%2Flio_rxtx.c;h=2bbb893c27e423e49d12b027ae3c450d40c9b381;hb=f37dfab21c988d2d0ecb3c82be4ba9738c7e51c7;hp=14113e1bef05fb41d76ba199778e89f8517a79e9;hpb=1f720275d887edaf7491a89634668de2af78b53e;p=dpdk.git

diff --git a/drivers/net/liquidio/lio_rxtx.c b/drivers/net/liquidio/lio_rxtx.c
index 14113e1bef..2bbb893c27 100644
--- a/drivers/net/liquidio/lio_rxtx.c
+++ b/drivers/net/liquidio/lio_rxtx.c
@@ -41,6 +41,9 @@
 #include "lio_rxtx.h"
 
 #define LIO_MAX_SG 12
+/* Flush iq if available tx_desc fall below LIO_FLUSH_WM */
+#define LIO_FLUSH_WM(_iq) ((_iq)->max_count / 2)
+#define LIO_PKT_IN_DONE_CNT_MASK 0x00000000FFFFFFFFULL
 
 static void
 lio_droq_compute_max_packet_bufs(struct lio_droq *droq)
@@ -78,28 +81,6 @@ lio_droq_destroy_ring_buffers(struct lio_droq *droq)
 	lio_droq_reset_indices(droq);
 }
 
-static void *
-lio_recv_buffer_alloc(struct lio_device *lio_dev, int q_no)
-{
-	struct lio_droq *droq = lio_dev->droq[q_no];
-	struct rte_mempool *mpool = droq->mpool;
-	struct rte_mbuf *m;
-
-	m = rte_pktmbuf_alloc(mpool);
-	if (m == NULL) {
-		lio_dev_err(lio_dev, "Cannot allocate\n");
-		return NULL;
-	}
-
-	rte_mbuf_refcnt_set(m, 1);
-	m->next = NULL;
-	m->data_off = RTE_PKTMBUF_HEADROOM;
-	m->nb_segs = 1;
-	m->pool = mpool;
-
-	return m;
-}
-
 static int
 lio_droq_setup_ring_buffers(struct lio_device *lio_dev,
 			    struct lio_droq *droq)
@@ -109,9 +90,10 @@ lio_droq_setup_ring_buffers(struct lio_device *lio_dev,
 	void *buf;
 
 	for (i = 0; i < droq->max_count; i++) {
-		buf = lio_recv_buffer_alloc(lio_dev, droq->q_no);
+		buf = rte_pktmbuf_alloc(droq->mpool);
 		if (buf == NULL) {
 			lio_dev_err(lio_dev, "buffer alloc failed\n");
+			droq->stats.rx_alloc_failure++;
 			lio_droq_destroy_ring_buffers(droq);
 			return -ENOMEM;
 		}
@@ -374,7 +356,6 @@ lio_droq_refill_pullup_descs(struct lio_droq *droq,
 
 /* lio_droq_refill
  *
- * @param lio_dev	- pointer to the lio device structure
  * @param droq		- droq in which descriptors require new buffers.
  *
  * Description:
@@ -390,7 +371,7 @@ lio_droq_refill_pullup_descs(struct lio_droq *droq,
  * This routine is called with droq->lock held.
  */
 static uint32_t
-lio_droq_refill(struct lio_device *lio_dev, struct lio_droq *droq)
+lio_droq_refill(struct lio_droq *droq)
 {
 	struct lio_droq_desc *desc_ring;
 	uint32_t desc_refilled = 0;
@@ -403,12 +384,14 @@ lio_droq_refill(struct lio_device *lio_dev, struct lio_droq *droq)
 		 * reuse the buffer, else allocate.
 		 */
 		if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) {
-			buf = lio_recv_buffer_alloc(lio_dev, droq->q_no);
+			buf = rte_pktmbuf_alloc(droq->mpool);
 			/* If a buffer could not be allocated, no point in
 			 * continuing
 			 */
-			if (buf == NULL)
+			if (buf == NULL) {
+				droq->stats.rx_alloc_failure++;
 				break;
+			}
 
 			droq->recv_buf_list[droq->refill_idx].buffer = buf;
 		}
@@ -483,9 +466,6 @@ lio_droq_fast_process_packet(struct lio_device *lio_dev,
 			droq->refill_count++;
 
 			if (likely(nicbuf != NULL)) {
-				nicbuf->data_off = RTE_PKTMBUF_HEADROOM;
-				nicbuf->nb_segs = 1;
-				nicbuf->next = NULL;
 				/* We don't have a way to pass flags yet */
 				nicbuf->ol_flags = 0;
 				if (rh->r_dh.has_hash) {
@@ -539,9 +519,6 @@ lio_droq_fast_process_packet(struct lio_device *lio_dev,
 					if (!pkt_len)
 						first_buf = nicbuf;
 
-					nicbuf->data_off = RTE_PKTMBUF_HEADROOM;
-					nicbuf->nb_segs = 1;
-					nicbuf->next = NULL;
 					nicbuf->port = lio_dev->port_id;
 					/* We don't have a way to pass
 					 * flags yet
@@ -611,7 +588,7 @@ lio_droq_fast_process_packet(struct lio_device *lio_dev,
 	}
 
 	if (droq->refill_count >= droq->refill_threshold) {
-		int desc_refilled = lio_droq_refill(lio_dev, droq);
+		int desc_refilled = lio_droq_refill(droq);
 
 		/* Flush the droq descriptor data to memory to be sure
 		 * that when we update the credits the data in memory is
@@ -626,6 +603,11 @@ lio_droq_fast_process_packet(struct lio_device *lio_dev,
 	info->length = 0;
 	info->rh.rh64 = 0;
 
+	droq->stats.pkts_received++;
+	droq->stats.rx_pkts_received += data_pkts;
+	droq->stats.rx_bytes_received += data_total_len;
+	droq->stats.bytes_received += total_len;
+
 	return data_pkts;
 }
 
@@ -907,6 +889,40 @@ release_lio_iq:
 	return -1;
 }
 
+int
+lio_wait_for_instr_fetch(struct lio_device *lio_dev)
+{
+	int pending, instr_cnt;
+	int i, retry = 1000;
+
+	do {
+		instr_cnt = 0;
+
+		for (i = 0; i < LIO_MAX_INSTR_QUEUES(lio_dev); i++) {
+			if (!(lio_dev->io_qmask.iq & (1ULL << i)))
+				continue;
+
+			if (lio_dev->instr_queue[i] == NULL)
+				break;
+
+			pending = rte_atomic64_read(
+			    &lio_dev->instr_queue[i]->instr_pending);
+			if (pending)
+				lio_flush_iq(lio_dev, lio_dev->instr_queue[i]);
+
+			instr_cnt += pending;
+		}
+
+		if (instr_cnt == 0)
+			break;
+
+		rte_delay_ms(1);
+
+	} while (retry-- && instr_cnt);
+
+	return instr_cnt;
+}
+
 static inline void
 lio_ring_doorbell(struct lio_device *lio_dev,
 		  struct lio_instr_queue *iq)
@@ -977,9 +993,151 @@ lio_add_to_request_list(struct lio_instr_queue *iq,
 	iq->request_list[idx].reqtype = reqtype;
 }
 
+static inline void
+lio_free_netsgbuf(void *buf)
+{
+	struct lio_buf_free_info *finfo = buf;
+	struct lio_device *lio_dev = finfo->lio_dev;
+	struct rte_mbuf *m = finfo->mbuf;
+	struct lio_gather *g = finfo->g;
+	uint8_t iq = finfo->iq_no;
+
+	/* This will take care of multiple segments also */
+	rte_pktmbuf_free(m);
+
+	rte_spinlock_lock(&lio_dev->glist_lock[iq]);
+	STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq], &g->list, entries);
+	rte_spinlock_unlock(&lio_dev->glist_lock[iq]);
+	rte_free(finfo);
+}
+
+/* Can only run in process context */
+static int
+lio_process_iq_request_list(struct lio_device *lio_dev,
+			    struct lio_instr_queue *iq)
+{
+	struct octeon_instr_irh *irh = NULL;
+	uint32_t old = iq->flush_index;
+	struct lio_soft_command *sc;
+	uint32_t inst_count = 0;
+	int reqtype;
+	void *buf;
+
+	while (old != iq->lio_read_index) {
+		reqtype = iq->request_list[old].reqtype;
+		buf     = iq->request_list[old].buf;
+
+		if (reqtype == LIO_REQTYPE_NONE)
+			goto skip_this;
+
+		switch (reqtype) {
+		case LIO_REQTYPE_NORESP_NET:
+			rte_pktmbuf_free((struct rte_mbuf *)buf);
+			break;
+		case LIO_REQTYPE_NORESP_NET_SG:
+			lio_free_netsgbuf(buf);
+			break;
+		case LIO_REQTYPE_SOFT_COMMAND:
+			sc = buf;
+			irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
+			if (irh->rflag) {
+				/* We're expecting a response from Octeon.
+				 * It's up to lio_process_ordered_list() to
+				 * process sc. Add sc to the ordered soft
+				 * command response list because we expect
+				 * a response from Octeon.
+				 */
+				rte_spinlock_lock(&lio_dev->response_list.lock);
+				rte_atomic64_inc(
+				    &lio_dev->response_list.pending_req_count);
+				STAILQ_INSERT_TAIL(
+					&lio_dev->response_list.head,
+					&sc->node, entries);
+				rte_spinlock_unlock(
+						&lio_dev->response_list.lock);
+			} else {
+				if (sc->callback) {
+					/* This callback must not sleep */
+					sc->callback(LIO_REQUEST_DONE,
+						     sc->callback_arg);
+				}
+			}
+			break;
+		default:
+			lio_dev_err(lio_dev,
+				    "Unknown reqtype: %d buf: %p at idx %d\n",
+				    reqtype, buf, old);
+		}
+
+		iq->request_list[old].buf = NULL;
+		iq->request_list[old].reqtype = 0;
+
+skip_this:
+		inst_count++;
+		old = lio_incr_index(old, 1, iq->max_count);
+	}
+
+	iq->flush_index = old;
+
+	return inst_count;
+}
+
+static void
+lio_update_read_index(struct lio_instr_queue *iq)
+{
+	uint32_t pkt_in_done = rte_read32(iq->inst_cnt_reg);
+	uint32_t last_done;
+
+	last_done = pkt_in_done - iq->pkt_in_done;
+	iq->pkt_in_done = pkt_in_done;
+
+	/* Add last_done and modulo with the IQ size to get new index */
+	iq->lio_read_index = (iq->lio_read_index +
+			(uint32_t)(last_done & LIO_PKT_IN_DONE_CNT_MASK)) %
+			iq->max_count;
+}
+
+int
+lio_flush_iq(struct lio_device *lio_dev, struct lio_instr_queue *iq)
+{
+	uint32_t tot_inst_processed = 0;
+	uint32_t inst_processed = 0;
+	int tx_done = 1;
+
+	if (rte_atomic64_test_and_set(&iq->iq_flush_running) == 0)
+		return tx_done;
+
+	rte_spinlock_lock(&iq->lock);
+
+	lio_update_read_index(iq);
+
+	do {
+		/* Process any outstanding IQ packets. */
+		if (iq->flush_index == iq->lio_read_index)
+			break;
+
+		inst_processed = lio_process_iq_request_list(lio_dev, iq);
+
+		if (inst_processed) {
+			rte_atomic64_sub(&iq->instr_pending, inst_processed);
+			iq->stats.instr_processed += inst_processed;
+		}
+
+		tot_inst_processed += inst_processed;
+		inst_processed = 0;
+
+	} while (1);
+
+	rte_spinlock_unlock(&iq->lock);
+
+	rte_atomic64_clear(&iq->iq_flush_running);
+
+	return tx_done;
+}
+
 static int
 lio_send_command(struct lio_device *lio_dev, uint32_t iq_no, void *cmd,
-		 void *buf, uint32_t datasize __rte_unused, uint32_t reqtype)
+		 void *buf, uint32_t datasize, uint32_t reqtype)
 {
 	struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no];
 	struct lio_iq_post_status st;
@@ -990,7 +1148,13 @@ lio_send_command(struct lio_device *lio_dev, uint32_t iq_no, void *cmd,
 
 	if (st.status != LIO_IQ_SEND_FAILED) {
 		lio_add_to_request_list(iq, st.index, buf, reqtype);
+		LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, bytes_sent,
+					      datasize);
+		LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_posted, 1);
+
 		lio_ring_doorbell(lio_dev, iq);
+	} else {
+		LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_dropped, 1);
 	}
 
 	rte_spinlock_unlock(&iq->post_lock);
@@ -1296,7 +1460,7 @@ list_delete_first_node(struct lio_stailq_head *head)
 	return node;
 }
 
-static void
+void
 lio_delete_sglist(struct lio_instr_queue *txq)
 {
 	struct lio_device *lio_dev = txq->lio_dev;
@@ -1385,6 +1549,117 @@ lio_delete_instruction_queue(struct lio_device *lio_dev, int iq_no)
 	lio_dev->num_iqs--;
 }
 
+static inline uint32_t
+lio_iq_get_available(struct lio_device *lio_dev, uint32_t q_no)
+{
+	return ((lio_dev->instr_queue[q_no]->max_count - 1) -
+		(uint32_t)rte_atomic64_read(
+				&lio_dev->instr_queue[q_no]->instr_pending));
+}
+
+static inline int
+lio_iq_is_full(struct lio_device *lio_dev, uint32_t q_no)
+{
+	return ((uint32_t)rte_atomic64_read(
+				&lio_dev->instr_queue[q_no]->instr_pending) >=
+				(lio_dev->instr_queue[q_no]->max_count - 2));
+}
+
+static int
+lio_dev_cleanup_iq(struct lio_device *lio_dev, int iq_no)
+{
+	struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no];
+	uint32_t count = 10000;
+
+	while ((lio_iq_get_available(lio_dev, iq_no) < LIO_FLUSH_WM(iq)) &&
+			--count)
+		lio_flush_iq(lio_dev, iq);
+
+	return count ? 0 : 1;
+}
+
+static void
+lio_ctrl_cmd_callback(uint32_t status __rte_unused, void *sc_ptr)
+{
+	struct lio_soft_command *sc = sc_ptr;
+	struct lio_dev_ctrl_cmd *ctrl_cmd;
+	struct lio_ctrl_pkt *ctrl_pkt;
+
+	ctrl_pkt = (struct lio_ctrl_pkt *)sc->ctxptr;
+	ctrl_cmd = ctrl_pkt->ctrl_cmd;
+	ctrl_cmd->cond = 1;
+
+	lio_free_soft_command(sc);
+}
+
+static inline struct lio_soft_command *
+lio_alloc_ctrl_pkt_sc(struct lio_device *lio_dev,
+		      struct lio_ctrl_pkt *ctrl_pkt)
+{
+	struct lio_soft_command *sc = NULL;
+	uint32_t uddsize, datasize;
+	uint32_t rdatasize;
+	uint8_t *data;
+
+	uddsize = (uint32_t)(ctrl_pkt->ncmd.s.more * 8);
+
+	datasize = OCTEON_CMD_SIZE + uddsize;
+	rdatasize = (ctrl_pkt->wait_time) ? 16 : 0;
+
+	sc = lio_alloc_soft_command(lio_dev, datasize,
+				    rdatasize, sizeof(struct lio_ctrl_pkt));
+	if (sc == NULL)
+		return NULL;
+
+	rte_memcpy(sc->ctxptr, ctrl_pkt, sizeof(struct lio_ctrl_pkt));
+
+	data = (uint8_t *)sc->virtdptr;
+
+	rte_memcpy(data, &ctrl_pkt->ncmd, OCTEON_CMD_SIZE);
+
+	lio_swap_8B_data((uint64_t *)data, OCTEON_CMD_SIZE >> 3);
+
+	if (uddsize) {
+		/* Endian-Swap for UDD should have been done by caller. */
+		rte_memcpy(data + OCTEON_CMD_SIZE, ctrl_pkt->udd, uddsize);
+	}
+
+	sc->iq_no = (uint32_t)ctrl_pkt->iq_no;
+
+	lio_prepare_soft_command(lio_dev, sc,
+				 LIO_OPCODE, LIO_OPCODE_CMD,
+				 0, 0, 0);
+
+	sc->callback = lio_ctrl_cmd_callback;
+	sc->callback_arg = sc;
+	sc->wait_time = ctrl_pkt->wait_time;
+
+	return sc;
+}
+
+int
+lio_send_ctrl_pkt(struct lio_device *lio_dev, struct lio_ctrl_pkt *ctrl_pkt)
+{
+	struct lio_soft_command *sc = NULL;
+	int retval;
+
+	sc = lio_alloc_ctrl_pkt_sc(lio_dev, ctrl_pkt);
+	if (sc == NULL) {
+		lio_dev_err(lio_dev, "soft command allocation failed\n");
+		return -1;
+	}
+
+	retval = lio_send_soft_command(lio_dev, sc);
+	if (retval == LIO_IQ_SEND_FAILED) {
+		lio_free_soft_command(sc);
+		lio_dev_err(lio_dev, "Port: %d soft command: %d send failed status: %x\n",
+			    lio_dev->port_id, ctrl_pkt->ncmd.s.cmd, retval);
+		return -1;
+	}
+
+	return retval;
+}
+
 /** Send data packet to the device
  *  @param lio_dev - lio device pointer
  *  @param ndata   - control structure with queueing, and buffer information
@@ -1405,6 +1680,7 @@ lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
 	struct lio_instr_queue *txq = tx_queue;
 	union lio_cmd_setup cmdsetup;
 	struct lio_device *lio_dev;
+	struct lio_iq_stats *stats;
 	struct lio_data_pkt ndata;
 	int i, processed = 0;
 	struct rte_mbuf *m;
@@ -1414,13 +1690,16 @@ lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
 
 	lio_dev = txq->lio_dev;
 	iq_no = txq->txpciq.s.q_no;
+	stats = &lio_dev->instr_queue[iq_no]->stats;
 
-	if (!lio_dev->linfo.link.s.link_up) {
+	if (!lio_dev->intf_open || !lio_dev->linfo.link.s.link_up) {
 		PMD_TX_LOG(lio_dev, ERR, "Transmit failed link_status : %d\n",
 			   lio_dev->linfo.link.s.link_up);
 		goto xmit_failed;
 	}
 
+	lio_dev_cleanup_iq(lio_dev, iq_no);
+
 	for (i = 0; i < nb_pkts; i++) {
 		uint32_t pkt_len = 0;
 
@@ -1432,6 +1711,15 @@ lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
 		ndata.buf = m;
 
 		ndata.q_no = iq_no;
+		if (lio_iq_is_full(lio_dev, ndata.q_no)) {
+			stats->tx_iq_busy++;
+			if (lio_dev_cleanup_iq(lio_dev, iq_no)) {
+				PMD_TX_LOG(lio_dev, ERR,
+					   "Transmit failed iq:%d full\n",
+					   ndata.q_no);
+				break;
+			}
+		}
 
 		cmdsetup.cmd_setup64 = 0;
 		cmdsetup.s.iq_no = iq_no;
@@ -1440,7 +1728,9 @@ lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
 		if (m->ol_flags & PKT_TX_IP_CKSUM)
 			cmdsetup.s.ip_csum = 1;
 
-		if ((m->ol_flags & PKT_TX_TCP_CKSUM) ||
+		if (m->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+			cmdsetup.s.tnl_csum = 1;
+		else if ((m->ol_flags & PKT_TX_TCP_CKSUM) ||
 				(m->ol_flags & PKT_TX_UDP_CKSUM))
 			cmdsetup.s.transport_csum = 1;
 
@@ -1451,6 +1741,68 @@ lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
 					    &cmdsetup, tag);
 			ndata.cmd.cmd3.dptr = rte_mbuf_data_dma_addr(m);
 			ndata.reqtype = LIO_REQTYPE_NORESP_NET;
+		} else {
+			struct lio_buf_free_info *finfo;
+			struct lio_gather *g;
+			phys_addr_t phyaddr;
+			int i, frags;
+
+			finfo = (struct lio_buf_free_info *)rte_malloc(NULL,
+							sizeof(*finfo), 0);
+			if (finfo == NULL) {
+				PMD_TX_LOG(lio_dev, ERR,
+					   "free buffer alloc failed\n");
+				goto xmit_failed;
+			}
+
+			rte_spinlock_lock(&lio_dev->glist_lock[iq_no]);
+			g = (struct lio_gather *)list_delete_first_node(
+						&lio_dev->glist_head[iq_no]);
+			rte_spinlock_unlock(&lio_dev->glist_lock[iq_no]);
+			if (g == NULL) {
+				PMD_TX_LOG(lio_dev, ERR,
+					   "Transmit scatter gather: glist null!\n");
+				goto xmit_failed;
+			}
+
+			cmdsetup.s.gather = 1;
+			cmdsetup.s.u.gatherptrs = m->nb_segs;
+			lio_prepare_pci_cmd(lio_dev, &ndata.cmd,
+					    &cmdsetup, tag);
+
+			memset(g->sg, 0, g->sg_size);
+			g->sg[0].ptr[0] = rte_mbuf_data_dma_addr(m);
+			lio_add_sg_size(&g->sg[0], m->data_len, 0);
+			pkt_len = m->data_len;
+			finfo->mbuf = m;
+
+			/* First seg taken care above */
+			frags = m->nb_segs - 1;
+			i = 1;
+			m = m->next;
+			while (frags--) {
+				g->sg[(i >> 2)].ptr[(i & 3)] =
+						rte_mbuf_data_dma_addr(m);
+				lio_add_sg_size(&g->sg[(i >> 2)],
+						m->data_len, (i & 3));
+				pkt_len += m->data_len;
+				i++;
+				m = m->next;
+			}
+
+			phyaddr = rte_mem_virt2phy(g->sg);
+			if (phyaddr == RTE_BAD_PHYS_ADDR) {
+				PMD_TX_LOG(lio_dev, ERR, "bad phys addr\n");
+				goto xmit_failed;
+			}
+
+			ndata.cmd.cmd3.dptr = phyaddr;
+			ndata.reqtype = LIO_REQTYPE_NORESP_NET_SG;
+
+			finfo->g = g;
+			finfo->lio_dev = lio_dev;
+			finfo->iq_no = (uint64_t)iq_no;
+			ndata.buf = finfo;
 		}
 
 		ndata.datasize = pkt_len;
@@ -1462,14 +1814,43 @@ lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
 			break;
 		}
 
-		if (unlikely(status == LIO_IQ_SEND_STOP))
+		if (unlikely(status == LIO_IQ_SEND_STOP)) {
 			PMD_TX_LOG(lio_dev, DEBUG, "iq full\n");
+			/* create space as iq is full */
+			lio_dev_cleanup_iq(lio_dev, iq_no);
+		}
 
+		stats->tx_done++;
+		stats->tx_tot_bytes += pkt_len;
 		processed++;
 	}
 
 xmit_failed:
+	stats->tx_dropped += (nb_pkts - processed);
 
 	return processed;
 }
 
+void
+lio_dev_clear_queues(struct rte_eth_dev *eth_dev)
+{
+	struct lio_instr_queue *txq;
+	struct lio_droq *rxq;
+	uint16_t i;
+
+	for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
+		txq = eth_dev->data->tx_queues[i];
+		if (txq != NULL) {
+			lio_dev_tx_queue_release(txq);
+			eth_dev->data->tx_queues[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
+		rxq = eth_dev->data->rx_queues[i];
+		if (rxq != NULL) {
+			lio_dev_rx_queue_release(rxq);
+			eth_dev->data->rx_queues[i] = NULL;
+		}
+	}
+}