From: Andrew Boyer <aboyer@pensando.io>
Date: Tue, 16 Feb 2021 20:35:33 +0000 (-0800)
Subject: net/ionic: split up queue-completion queue structure
X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=be39f75cd47f590e801912c085f895d6f8143ec1;p=dpdk.git

net/ionic: split up queue-completion queue structure

Create a unique Q-CQ struct for adminq, notifyq, rxq, and txq to
reduce the size of each object.

Minimize the size of each field to squeeze into as few cachelines
as possible.

Signed-off-by: Andrew Boyer <aboyer@pensando.io>
---

diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c
index fcd6fca9a3..87579a09e5 100644
--- a/drivers/net/ionic/ionic_lif.c
+++ b/drivers/net/ionic/ionic_lif.c
@@ -76,13 +76,13 @@ ionic_lif_stop(struct ionic_lif *lif)
 	lif->state &= ~IONIC_LIF_F_UP;
 
 	for (i = 0; i < lif->nrxqcqs; i++) {
-		struct ionic_qcq *rxq = lif->rxqcqs[i];
+		struct ionic_rx_qcq *rxq = lif->rxqcqs[i];
 		if (rxq->flags & IONIC_QCQ_F_INITED)
 			(void)ionic_dev_rx_queue_stop(lif->eth_dev, i);
 	}
 
 	for (i = 0; i < lif->ntxqcqs; i++) {
-		struct ionic_qcq *txq = lif->txqcqs[i];
+		struct ionic_tx_qcq *txq = lif->txqcqs[i];
 		if (txq->flags & IONIC_QCQ_F_INITED)
 			(void)ionic_dev_tx_queue_stop(lif->eth_dev, i);
 	}
@@ -131,7 +131,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats
 		ls->rx_bcast_bytes;
 
 	for (i = 0; i < lif->nrxqcqs; i++) {
-		struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
+		struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats;
 		stats->imissed +=
 			rx_stats->no_cb_arg +
 			rx_stats->bad_cq_status +
@@ -152,7 +152,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats
 		ls->rx_desc_data_error;
 
 	for (i = 0; i < num_rx_q_counters; i++) {
-		struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
+		struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats;
 		stats->q_ipackets[i] = rx_stats->packets;
 		stats->q_ibytes[i] = rx_stats->bytes;
 		stats->q_errors[i] =
@@ -173,7 +173,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats
 		ls->tx_bcast_bytes;
 
 	for (i = 0; i < lif->ntxqcqs; i++) {
-		struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
+		struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats;
 		stats->oerrors += tx_stats->drop;
 	}
 
@@ -189,7 +189,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats
 		ls->tx_desc_data_error;
 
 	for (i = 0; i < num_tx_q_counters; i++) {
-		struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
+		struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats;
 		stats->q_opackets[i] = tx_stats->packets;
 		stats->q_obytes[i] = tx_stats->bytes;
 	}
@@ -217,9 +217,9 @@ ionic_lif_reset_stats(struct ionic_lif *lif)
 	uint32_t i;
 
 	for (i = 0; i < lif->nrxqcqs; i++) {
-		memset(&lif->rxqcqs[i]->stats.rx, 0,
+		memset(&lif->rxqcqs[i]->stats, 0,
 			sizeof(struct ionic_rx_stats));
-		memset(&lif->txqcqs[i]->stats.tx, 0,
+		memset(&lif->txqcqs[i]->stats, 0,
 			sizeof(struct ionic_tx_stats));
 	}
 
@@ -587,6 +587,7 @@ ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 static int
 ionic_qcq_alloc(struct ionic_lif *lif,
 		uint8_t type,
+		size_t struct_size,
 		uint32_t index,
 		const char *type_name,
 		uint16_t flags,
@@ -625,16 +626,17 @@ ionic_qcq_alloc(struct ionic_lif *lif,
 		total_size += PAGE_SIZE;
 	}
 
-	new = rte_zmalloc("ionic", sizeof(*new), 0);
+	new = rte_zmalloc("ionic", struct_size, 0);
 	if (!new) {
 		IONIC_PRINT(ERR, "Cannot allocate queue structure");
 		return -ENOMEM;
 	}
 
 	new->lif = lif;
-	new->flags = flags;
 
-	new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
+	new->q.info = rte_calloc_socket("ionic",
+				num_descs, sizeof(void *),
+				PAGE_SIZE, socket_id);
 	if (!new->q.info) {
 		IONIC_PRINT(ERR, "Cannot allocate queue info");
 		err = -ENOMEM;
@@ -667,7 +669,6 @@ ionic_qcq_alloc(struct ionic_lif *lif,
 
 	new->base = new->base_z->addr;
 	new->base_pa = new->base_z->iova;
-	new->total_size = total_size;
 
 	q_base = new->base;
 	q_base_pa = new->base_pa;
@@ -720,15 +721,17 @@ ionic_qcq_free(struct ionic_qcq *qcq)
 }
 
 int
-ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs,
-		struct ionic_qcq **qcq)
+ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index,
+		uint16_t nrxq_descs, struct ionic_rx_qcq **rxq_out)
 {
-	uint32_t flags;
-	int err = -ENOMEM;
+	struct ionic_rx_qcq *rxq;
+	uint16_t flags;
+	int err;
 
 	flags = IONIC_QCQ_F_SG;
 	err = ionic_qcq_alloc(lif,
 		IONIC_QTYPE_RXQ,
+		sizeof(struct ionic_rx_qcq),
 		index,
 		"rx",
 		flags,
@@ -736,25 +739,30 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs,
 		sizeof(struct ionic_rxq_desc),
 		sizeof(struct ionic_rxq_comp),
 		sizeof(struct ionic_rxq_sg_desc),
-		&lif->rxqcqs[index]);
+		(struct ionic_qcq **)&rxq);
 	if (err)
 		return err;
 
-	*qcq = lif->rxqcqs[index];
+	rxq->flags = flags;
+
+	lif->rxqcqs[index] = rxq;
+	*rxq_out = rxq;
 
 	return 0;
 }
 
 int
-ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
-		struct ionic_qcq **qcq)
+ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index,
+		uint16_t ntxq_descs, struct ionic_tx_qcq **txq_out)
 {
-	uint32_t flags;
-	int err = -ENOMEM;
+	struct ionic_tx_qcq *txq;
+	uint16_t flags;
+	int err;
 
 	flags = IONIC_QCQ_F_SG;
 	err = ionic_qcq_alloc(lif,
 		IONIC_QTYPE_TXQ,
+		sizeof(struct ionic_tx_qcq),
 		index,
 		"tx",
 		flags,
@@ -762,11 +770,14 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
 		sizeof(struct ionic_txq_desc),
 		sizeof(struct ionic_txq_comp),
 		sizeof(struct ionic_txq_sg_desc_v1),
-		&lif->txqcqs[index]);
+		(struct ionic_qcq **)&txq);
 	if (err)
 		return err;
 
-	*qcq = lif->txqcqs[index];
+	txq->flags = flags;
+
+	lif->txqcqs[index] = txq;
+	*txq_out = txq;
 
 	return 0;
 }
@@ -774,12 +785,12 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
 static int
 ionic_admin_qcq_alloc(struct ionic_lif *lif)
 {
-	uint32_t flags;
-	int err = -ENOMEM;
+	uint16_t flags = 0;
+	int err;
 
-	flags = 0;
 	err = ionic_qcq_alloc(lif,
 		IONIC_QTYPE_ADMINQ,
+		sizeof(struct ionic_admin_qcq),
 		0,
 		"admin",
 		flags,
@@ -787,7 +798,7 @@ ionic_admin_qcq_alloc(struct ionic_lif *lif)
 		sizeof(struct ionic_admin_cmd),
 		sizeof(struct ionic_admin_comp),
 		0,
-		&lif->adminqcq);
+		(struct ionic_qcq **)&lif->adminqcq);
 	if (err)
 		return err;
 
@@ -797,13 +808,14 @@ ionic_admin_qcq_alloc(struct ionic_lif *lif)
 static int
 ionic_notify_qcq_alloc(struct ionic_lif *lif)
 {
-	struct ionic_qcq *nqcq;
+	struct ionic_notify_qcq *nqcq;
 	struct ionic_dev *idev = &lif->adapter->idev;
-	uint32_t flags = 0;
-	int err = -ENOMEM;
+	uint16_t flags = 0;
+	int err;
 
 	err = ionic_qcq_alloc(lif,
 		IONIC_QTYPE_NOTIFYQ,
+		sizeof(struct ionic_notify_qcq),
 		0,
 		"notify",
 		flags,
@@ -811,13 +823,13 @@ ionic_notify_qcq_alloc(struct ionic_lif *lif)
 		sizeof(struct ionic_notifyq_cmd),
 		sizeof(union ionic_notifyq_comp),
 		0,
-		&nqcq);
+		(struct ionic_qcq **)&nqcq);
 	if (err)
 		return err;
 
 	err = ionic_intr_alloc(lif, &nqcq->intr);
 	if (err) {
-		ionic_qcq_free(nqcq);
+		ionic_qcq_free(&nqcq->qcq);
 		return err;
 	}
 
@@ -1002,12 +1014,12 @@ void
 ionic_lif_free(struct ionic_lif *lif)
 {
 	if (lif->notifyqcq) {
-		ionic_qcq_free(lif->notifyqcq);
+		ionic_qcq_free(&lif->notifyqcq->qcq);
 		lif->notifyqcq = NULL;
 	}
 
 	if (lif->adminqcq) {
-		ionic_qcq_free(lif->adminqcq);
+		ionic_qcq_free(&lif->adminqcq->qcq);
 		lif->adminqcq = NULL;
 	}
 
@@ -1137,28 +1149,28 @@ ionic_lif_rss_teardown(struct ionic_lif *lif)
 	}
 }
 
-static void
-ionic_lif_qcq_deinit(struct ionic_qcq *qcq)
+void
+ionic_lif_txq_deinit(struct ionic_tx_qcq *txq)
 {
-	qcq->flags &= ~IONIC_QCQ_F_INITED;
+	txq->flags &= ~IONIC_QCQ_F_INITED;
 }
 
 void
-ionic_lif_txq_deinit(struct ionic_qcq *qcq)
+ionic_lif_rxq_deinit(struct ionic_rx_qcq *rxq)
 {
-	ionic_lif_qcq_deinit(qcq);
+	rxq->flags &= ~IONIC_QCQ_F_INITED;
 }
 
-void
-ionic_lif_rxq_deinit(struct ionic_qcq *qcq)
+static void
+ionic_lif_adminq_deinit(struct ionic_lif *lif)
 {
-	ionic_lif_qcq_deinit(qcq);
+	lif->adminqcq->flags &= ~IONIC_QCQ_F_INITED;
 }
 
 static void
 ionic_lif_notifyq_deinit(struct ionic_lif *lif)
 {
-	struct ionic_qcq *nqcq = lif->notifyqcq;
+	struct ionic_notify_qcq *nqcq = lif->notifyqcq;
 	struct ionic_dev *idev = &lif->adapter->idev;
 
 	if (!(nqcq->flags & IONIC_QCQ_F_INITED))
@@ -1283,26 +1295,27 @@ int
 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
 {
 	struct ionic_dev *idev = &lif->adapter->idev;
-	struct ionic_qcq *qcq = lif->notifyqcq;
+	struct ionic_notify_qcq *nqcq = lif->notifyqcq;
 	uint32_t work_done;
 
-	if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
+	if (!(nqcq->flags & IONIC_QCQ_F_INITED)) {
 		IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
 		return -1;
 	}
 
-	ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
+	ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index,
 		IONIC_INTR_MASK_SET);
 
-	work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
+	work_done = ionic_qcq_service(&nqcq->qcq, budget,
+				ionic_notifyq_cb, lif);
 
 	if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
 		ionic_link_status_check(lif);
 
-	ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
+	ionic_intr_credits(idev->intr_ctrl, nqcq->intr.index,
 		work_done, IONIC_INTR_CRED_RESET_COALESCE);
 
-	ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
+	ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index,
 		IONIC_INTR_MASK_CLEAR);
 
 	return 0;
@@ -1312,12 +1325,12 @@ static int
 ionic_lif_adminq_init(struct ionic_lif *lif)
 {
 	struct ionic_dev *idev = &lif->adapter->idev;
-	struct ionic_qcq *qcq = lif->adminqcq;
-	struct ionic_queue *q = &qcq->q;
+	struct ionic_admin_qcq *aqcq = lif->adminqcq;
+	struct ionic_queue *q = &aqcq->qcq.q;
 	struct ionic_q_init_comp comp;
 	int err;
 
-	ionic_dev_cmd_adminq_init(idev, qcq);
+	ionic_dev_cmd_adminq_init(idev, &aqcq->qcq);
 	err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
 	if (err)
 		return err;
@@ -1332,7 +1345,7 @@ ionic_lif_adminq_init(struct ionic_lif *lif)
 	IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
 	IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
 
-	qcq->flags |= IONIC_QCQ_F_INITED;
+	aqcq->flags |= IONIC_QCQ_F_INITED;
 
 	return 0;
 }
@@ -1341,8 +1354,8 @@ static int
 ionic_lif_notifyq_init(struct ionic_lif *lif)
 {
 	struct ionic_dev *idev = &lif->adapter->idev;
-	struct ionic_qcq *qcq = lif->notifyqcq;
-	struct ionic_queue *q = &qcq->q;
+	struct ionic_notify_qcq *nqcq = lif->notifyqcq;
+	struct ionic_queue *q = &nqcq->qcq.q;
 	int err;
 
 	struct ionic_admin_ctx ctx = {
@@ -1352,7 +1365,7 @@ ionic_lif_notifyq_init(struct ionic_lif *lif)
 			.type = q->type,
 			.ver = lif->qtype_info[q->type].version,
 			.index = rte_cpu_to_le_32(q->index),
-			.intr_index = rte_cpu_to_le_16(qcq->intr.index),
+			.intr_index = rte_cpu_to_le_16(nqcq->intr.index),
 			.flags = rte_cpu_to_le_16(IONIC_QINIT_F_IRQ |
 						IONIC_QINIT_F_ENA),
 			.ring_size = rte_log2_u32(q->num_descs),
@@ -1378,10 +1391,10 @@ ionic_lif_notifyq_init(struct ionic_lif *lif)
 	IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
 	IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
 
-	ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
+	ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index,
 		IONIC_INTR_MASK_CLEAR);
 
-	qcq->flags |= IONIC_QCQ_F_INITED;
+	nqcq->flags |= IONIC_QCQ_F_INITED;
 
 	return 0;
 }
@@ -1445,8 +1458,9 @@ ionic_lif_set_features(struct ionic_lif *lif)
 }
 
 int
-ionic_lif_txq_init(struct ionic_qcq *qcq)
+ionic_lif_txq_init(struct ionic_tx_qcq *txq)
 {
+	struct ionic_qcq *qcq = &txq->qcq;
 	struct ionic_queue *q = &qcq->q;
 	struct ionic_lif *lif = qcq->lif;
 	struct ionic_cq *cq = &qcq->cq;
@@ -1474,7 +1488,7 @@ ionic_lif_txq_init(struct ionic_qcq *qcq)
 		ctx.cmd.q_init.ring_size);
 	IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver);
 
-	err = ionic_adminq_post_wait(qcq->lif, &ctx);
+	err = ionic_adminq_post_wait(lif, &ctx);
 	if (err)
 		return err;
 
@@ -1486,14 +1500,15 @@ ionic_lif_txq_init(struct ionic_qcq *qcq)
 	IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index);
 	IONIC_PRINT(DEBUG, "txq->db %p", q->db);
 
-	qcq->flags |= IONIC_QCQ_F_INITED;
+	txq->flags |= IONIC_QCQ_F_INITED;
 
 	return 0;
 }
 
 int
-ionic_lif_rxq_init(struct ionic_qcq *qcq)
+ionic_lif_rxq_init(struct ionic_rx_qcq *rxq)
 {
+	struct ionic_qcq *qcq = &rxq->qcq;
 	struct ionic_queue *q = &qcq->q;
 	struct ionic_lif *lif = qcq->lif;
 	struct ionic_cq *cq = &qcq->cq;
@@ -1521,7 +1536,7 @@ ionic_lif_rxq_init(struct ionic_qcq *qcq)
 		ctx.cmd.q_init.ring_size);
 	IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver);
 
-	err = ionic_adminq_post_wait(qcq->lif, &ctx);
+	err = ionic_adminq_post_wait(lif, &ctx);
 	if (err)
 		return err;
 
@@ -1529,7 +1544,7 @@ ionic_lif_rxq_init(struct ionic_qcq *qcq)
 	q->hw_index = rte_le_to_cpu_32(ctx.comp.q_init.hw_index);
 	q->db = ionic_db_map(lif, q);
 
-	qcq->flags |= IONIC_QCQ_F_INITED;
+	rxq->flags |= IONIC_QCQ_F_INITED;
 
 	IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type);
 	IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index);
@@ -1634,7 +1649,7 @@ err_out_notifyq_deinit:
 	ionic_lif_notifyq_deinit(lif);
 
 err_out_adminq_deinit:
-	ionic_lif_qcq_deinit(lif->adminqcq);
+	ionic_lif_adminq_deinit(lif);
 
 	return err;
 }
@@ -1648,7 +1663,7 @@ ionic_lif_deinit(struct ionic_lif *lif)
 	ionic_rx_filters_deinit(lif);
 	ionic_lif_rss_teardown(lif);
 	ionic_lif_notifyq_deinit(lif);
-	ionic_lif_qcq_deinit(lif->adminqcq);
+	ionic_lif_adminq_deinit(lif);
 
 	lif->state &= ~IONIC_LIF_F_INITED;
 }
@@ -1788,7 +1803,7 @@ ionic_lif_start(struct ionic_lif *lif)
 		lif->nrxqcqs, lif->ntxqcqs, lif->port_id);
 
 	for (i = 0; i < lif->nrxqcqs; i++) {
-		struct ionic_qcq *rxq = lif->rxqcqs[i];
+		struct ionic_rx_qcq *rxq = lif->rxqcqs[i];
 		if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) {
 			err = ionic_dev_rx_queue_start(lif->eth_dev, i);
 
@@ -1798,7 +1813,7 @@ ionic_lif_start(struct ionic_lif *lif)
 	}
 
 	for (i = 0; i < lif->ntxqcqs; i++) {
-		struct ionic_qcq *txq = lif->txqcqs[i];
+		struct ionic_tx_qcq *txq = lif->txqcqs[i];
 		if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) {
 			err = ionic_dev_tx_queue_start(lif->eth_dev, i);
 
diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h
index c850a9c08d..a8243ebf21 100644
--- a/drivers/net/ionic/ionic_lif.h
+++ b/drivers/net/ionic/ionic_lif.h
@@ -58,23 +58,47 @@ struct ionic_qcq {
 	struct ionic_queue q;        /**< Queue */
 	struct ionic_cq cq;          /**< Completion Queue */
 	struct ionic_lif *lif;       /**< LIF */
-	struct rte_mempool *mb_pool; /**< mbuf pool to populate the RX ring */
-	union {
-		struct ionic_tx_stats tx;
-		struct ionic_rx_stats rx;
-	} stats;
 	const struct rte_memzone *base_z;
 	void *base;
 	rte_iova_t base_pa;
-	uint32_t total_size;
-	uint32_t flags;
+};
+
+struct ionic_admin_qcq {
+	struct ionic_qcq qcq;
+	uint16_t flags;
+};
+
+struct ionic_notify_qcq {
+	struct ionic_qcq qcq;
+	uint16_t flags;
+
 	struct ionic_intr_info intr;
 };
 
+struct ionic_rx_qcq {
+	/* cacheline0, cacheline1 */
+	struct ionic_qcq qcq;
+
+	/* cacheline2 */
+	struct rte_mempool *mb_pool;
+	uint16_t flags;
+
+	/* cacheline3 (inside stats) */
+	struct ionic_rx_stats stats;
+};
+
+struct ionic_tx_qcq {
+	/* cacheline0, cacheline1 */
+	struct ionic_qcq qcq;
+
+	/* cacheline2 */
+	uint16_t flags;
+
+	struct ionic_tx_stats stats;
+};
+
 #define IONIC_Q_TO_QCQ(_q)	container_of(_q, struct ionic_qcq, q)
 #define IONIC_CQ_TO_QCQ(_cq)	container_of(_cq, struct ionic_qcq, cq)
-#define IONIC_Q_TO_TX_STATS(q)	(&IONIC_Q_TO_QCQ(q)->stats.tx)
-#define IONIC_Q_TO_RX_STATS(q)	(&IONIC_Q_TO_QCQ(q)->stats.rx)
 
 struct ionic_qtype_info {
 	uint8_t  version;
@@ -104,10 +128,10 @@ struct ionic_lif {
 	uint32_t nrxqcqs;
 	rte_spinlock_t adminq_lock;
 	rte_spinlock_t adminq_service_lock;
-	struct ionic_qcq *adminqcq;
-	struct ionic_qcq *notifyqcq;
-	struct ionic_qcq **txqcqs;
-	struct ionic_qcq **rxqcqs;
+	struct ionic_admin_qcq *adminqcq;
+	struct ionic_notify_qcq *notifyqcq;
+	struct ionic_tx_qcq **txqcqs;
+	struct ionic_rx_qcq **rxqcqs;
 	struct ionic_rx_filters rx_filters;
 	struct ionic_doorbell __iomem *kern_dbpage;
 	uint64_t last_eid;
@@ -173,19 +197,19 @@ int ionic_dev_allmulticast_enable(struct rte_eth_dev *dev);
 int ionic_dev_allmulticast_disable(struct rte_eth_dev *dev);
 
 int ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index,
-	uint16_t nrxq_descs, struct ionic_qcq **qcq);
+	uint16_t nrxq_descs, struct ionic_rx_qcq **qcq_out);
 int ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index,
-	uint16_t ntxq_descs, struct ionic_qcq **qcq);
+	uint16_t ntxq_descs, struct ionic_tx_qcq **qcq_out);
 void ionic_qcq_free(struct ionic_qcq *qcq);
 
 int ionic_qcq_enable(struct ionic_qcq *qcq);
 int ionic_qcq_disable(struct ionic_qcq *qcq);
 
-int ionic_lif_rxq_init(struct ionic_qcq *qcq);
-void ionic_lif_rxq_deinit(struct ionic_qcq *qcq);
+int ionic_lif_rxq_init(struct ionic_rx_qcq *rxq);
+void ionic_lif_rxq_deinit(struct ionic_rx_qcq *rxq);
 
-int ionic_lif_txq_init(struct ionic_qcq *qcq);
-void ionic_lif_txq_deinit(struct ionic_qcq *qcq);
+int ionic_lif_txq_init(struct ionic_tx_qcq *txq);
+void ionic_lif_txq_deinit(struct ionic_tx_qcq *txq);
 
 int ionic_lif_rss_config(struct ionic_lif *lif, const uint16_t types,
 	const uint8_t *key, const uint32_t *indir);
diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c
index 167a97dcf4..0d2e02fdd0 100644
--- a/drivers/net/ionic/ionic_main.c
+++ b/drivers/net/ionic/ionic_main.c
@@ -194,7 +194,7 @@ ionic_adminq_service(struct ionic_cq *cq, uint16_t cq_desc_index,
 static int
 ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
 {
-	struct ionic_queue *q = &lif->adminqcq->q;
+	struct ionic_queue *q = &lif->adminqcq->qcq.q;
 	struct ionic_admin_cmd *q_desc_base = q->base;
 	struct ionic_admin_cmd *q_desc;
 	int err = 0;
@@ -234,7 +234,7 @@ ionic_adminq_wait_for_completion(struct ionic_lif *lif,
 		 */
 		rte_spinlock_lock(&lif->adminq_service_lock);
 
-		ionic_qcq_service(lif->adminqcq, budget,
+		ionic_qcq_service(&lif->adminqcq->qcq, budget,
 				ionic_adminq_service, NULL);
 
 		rte_spinlock_unlock(&lif->adminq_service_lock);
diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c
index 6ecb500b9e..5f836f0134 100644
--- a/drivers/net/ionic/ionic_rxtx.c
+++ b/drivers/net/ionic/ionic_rxtx.c
@@ -59,8 +59,8 @@ void
 ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 		struct rte_eth_txq_info *qinfo)
 {
-	struct ionic_qcq *txq = dev->data->tx_queues[queue_id];
-	struct ionic_queue *q = &txq->q;
+	struct ionic_tx_qcq *txq = dev->data->tx_queues[queue_id];
+	struct ionic_queue *q = &txq->qcq.q;
 
 	qinfo->nb_desc = q->num_descs;
 	qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
@@ -68,10 +68,10 @@ ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 }
 
 static __rte_always_inline void
-ionic_tx_flush(struct ionic_qcq *txq)
+ionic_tx_flush(struct ionic_tx_qcq *txq)
 {
-	struct ionic_cq *cq = &txq->cq;
-	struct ionic_queue *q = &txq->q;
+	struct ionic_cq *cq = &txq->qcq.cq;
+	struct ionic_queue *q = &txq->qcq.q;
 	struct rte_mbuf *txm, *next;
 	struct ionic_txq_comp *cq_desc_base = cq->base;
 	struct ionic_txq_comp *cq_desc;
@@ -122,19 +122,19 @@ ionic_tx_flush(struct ionic_qcq *txq)
 void __rte_cold
 ionic_dev_tx_queue_release(void *tx_queue)
 {
-	struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue;
+	struct ionic_tx_qcq *txq = tx_queue;
 
 	IONIC_PRINT_CALL();
 
 	ionic_lif_txq_deinit(txq);
 
-	ionic_qcq_free(txq);
+	ionic_qcq_free(&txq->qcq);
 }
 
 int __rte_cold
 ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 {
-	struct ionic_qcq *txq;
+	struct ionic_tx_qcq *txq;
 
 	IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id);
 
@@ -148,7 +148,7 @@ ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 	 * before disabling Tx queue
 	 */
 
-	ionic_qcq_disable(txq);
+	ionic_qcq_disable(&txq->qcq);
 
 	ionic_tx_flush(txq);
 
@@ -161,7 +161,7 @@ ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id,
 		const struct rte_eth_txconf *tx_conf)
 {
 	struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
-	struct ionic_qcq *txq;
+	struct ionic_tx_qcq *txq;
 	uint64_t offloads;
 	int err;
 
@@ -221,7 +221,7 @@ int __rte_cold
 ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 {
 	uint8_t *tx_queue_state = eth_dev->data->tx_queue_state;
-	struct ionic_qcq *txq;
+	struct ionic_tx_qcq *txq;
 	int err;
 
 	if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) {
@@ -233,14 +233,14 @@ ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 	txq = eth_dev->data->tx_queues[tx_queue_id];
 
 	IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs",
-		tx_queue_id, txq->q.num_descs);
+		tx_queue_id, txq->qcq.q.num_descs);
 
 	if (!(txq->flags & IONIC_QCQ_F_INITED)) {
 		err = ionic_lif_txq_init(txq);
 		if (err)
 			return err;
 	} else {
-		ionic_qcq_enable(txq);
+		ionic_qcq_enable(&txq->qcq);
 	}
 
 	tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
@@ -315,8 +315,9 @@ ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
 }
 
 static struct ionic_txq_desc *
-ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem)
+ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem)
 {
+	struct ionic_queue *q = &txq->qcq.q;
 	struct ionic_txq_desc *desc_base = q->base;
 	struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base;
 	struct ionic_txq_desc *desc = &desc_base[q->head_idx];
@@ -327,11 +328,11 @@ ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem)
 }
 
 static int
-ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm,
+ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm,
 		bool not_xmit_more)
 {
-	struct ionic_queue *q = &txq->q;
-	struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);
+	struct ionic_queue *q = &txq->qcq.q;
+	struct ionic_tx_stats *stats = &txq->stats;
 	struct ionic_txq_desc *desc;
 	struct ionic_txq_sg_elem *elem;
 	struct rte_mbuf *txm_seg;
@@ -375,7 +376,7 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm,
 	left = txm->data_len;
 	data_iova = rte_mbuf_data_iova(txm);
 
-	desc = ionic_tx_tso_next(q, &elem);
+	desc = ionic_tx_tso_next(txq, &elem);
 	start = true;
 
 	/* Chop data up into desc segments */
@@ -397,7 +398,7 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm,
 			encap,
 			vlan_tci, has_vlan,
 			start, done && not_xmit_more);
-		desc = ionic_tx_tso_next(q, &elem);
+		desc = ionic_tx_tso_next(txq, &elem);
 		start = false;
 		seglen = mss;
 	}
@@ -439,7 +440,7 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm,
 				encap,
 				vlan_tci, has_vlan,
 				start, done && not_xmit_more);
-			desc = ionic_tx_tso_next(q, &elem);
+			desc = ionic_tx_tso_next(txq, &elem);
 			start = false;
 		}
 
@@ -452,16 +453,14 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm,
 }
 
 static __rte_always_inline int
-ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm,
+ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm,
 		bool not_xmit_more)
 {
-	struct ionic_queue *q = &txq->q;
-	struct ionic_txq_desc *desc_base = q->base;
+	struct ionic_queue *q = &txq->qcq.q;
+	struct ionic_txq_desc *desc, *desc_base = q->base;
 	struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base;
-	struct ionic_txq_desc *desc = &desc_base[q->head_idx];
-	struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx];
-	struct ionic_txq_sg_elem *elem = sg_desc->elems;
-	struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);
+	struct ionic_txq_sg_elem *elem;
+	struct ionic_tx_stats *stats = &txq->stats;
 	struct rte_mbuf *txm_seg;
 	bool encap;
 	bool has_vlan;
@@ -470,6 +469,8 @@ ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm,
 	uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
 	uint8_t flags = 0;
 
+	desc = &desc_base[q->head_idx];
+
 	if ((ol_flags & PKT_TX_IP_CKSUM) &&
 	    (txq->flags & IONIC_QCQ_F_CSUM_L3)) {
 		opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
@@ -502,6 +503,7 @@ ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm,
 	desc->len = txm->data_len;
 	desc->vlan_tci = txm->vlan_tci;
 
+	elem = sg_desc_base[q->head_idx].elems;
 	txm_seg = txm->next;
 	while (txm_seg != NULL) {
 		elem->len = txm_seg->data_len;
@@ -520,9 +522,9 @@ uint16_t
 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		uint16_t nb_pkts)
 {
-	struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue;
-	struct ionic_queue *q = &txq->q;
-	struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);
+	struct ionic_tx_qcq *txq = tx_queue;
+	struct ionic_queue *q = &txq->qcq.q;
+	struct ionic_tx_stats *stats = &txq->stats;
 	uint32_t next_q_head_idx;
 	uint32_t bytes_tx = 0;
 	uint16_t nb_tx = 0;
@@ -625,8 +627,8 @@ void
 ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 		struct rte_eth_rxq_info *qinfo)
 {
-	struct ionic_qcq *rxq = dev->data->rx_queues[queue_id];
-	struct ionic_queue *q = &rxq->q;
+	struct ionic_rx_qcq *rxq = dev->data->rx_queues[queue_id];
+	struct ionic_queue *q = &rxq->qcq.q;
 
 	qinfo->mp = rxq->mb_pool;
 	qinfo->scattered_rx = dev->data->scattered_rx;
@@ -636,9 +638,9 @@ ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 }
 
 static void __rte_cold
-ionic_rx_empty(struct ionic_queue *q)
+ionic_rx_empty(struct ionic_rx_qcq *rxq)
 {
-	struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q);
+	struct ionic_queue *q = &rxq->qcq.q;
 	struct rte_mbuf *mbuf;
 	void **info;
 
@@ -654,15 +656,18 @@ ionic_rx_empty(struct ionic_queue *q)
 void __rte_cold
 ionic_dev_rx_queue_release(void *rx_queue)
 {
-	struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue;
+	struct ionic_rx_qcq *rxq = rx_queue;
+
+	if (!rxq)
+		return;
 
 	IONIC_PRINT_CALL();
 
-	ionic_rx_empty(&rxq->q);
+	ionic_rx_empty(rxq);
 
 	ionic_lif_rxq_deinit(rxq);
 
-	ionic_qcq_free(rxq);
+	ionic_qcq_free(&rxq->qcq);
 }
 
 int __rte_cold
@@ -674,7 +679,7 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 		struct rte_mempool *mp)
 {
 	struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
-	struct ionic_qcq *rxq;
+	struct ionic_rx_qcq *rxq;
 	uint64_t offloads;
 	int err;
 
@@ -713,7 +718,8 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 	eth_dev->data->rx_queue_state[rx_queue_id] =
 		RTE_ETH_QUEUE_STATE_STOPPED;
 
-	err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, &rxq);
+	err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc,
+			&rxq);
 	if (err) {
 		IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id);
 		return -EINVAL;
@@ -741,20 +747,20 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 }
 
 static __rte_always_inline void
-ionic_rx_clean(struct ionic_qcq *rxq,
+ionic_rx_clean(struct ionic_rx_qcq *rxq,
 		uint32_t q_desc_index, uint32_t cq_desc_index,
 		void *service_cb_arg)
 {
-	struct ionic_queue *q = &rxq->q;
-	struct ionic_cq *cq = &rxq->cq;
+	struct ionic_queue *q = &rxq->qcq.q;
+	struct ionic_cq *cq = &rxq->qcq.cq;
 	struct ionic_rxq_comp *cq_desc_base = cq->base;
 	struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index];
 	struct rte_mbuf *rxm, *rxm_seg;
 	uint32_t max_frame_size =
-		rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
+		rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 	uint64_t pkt_flags = 0;
 	uint32_t pkt_type;
-	struct ionic_rx_stats *stats = IONIC_Q_TO_RX_STATS(q);
+	struct ionic_rx_stats *stats = &rxq->stats;
 	struct ionic_rx_service *recv_args = (struct ionic_rx_service *)
 		service_cb_arg;
 	uint32_t buf_size = (uint16_t)
@@ -803,7 +809,7 @@ ionic_rx_clean(struct ionic_qcq *rxq,
 	rte_prefetch1((char *)rxm->buf_addr + rxm->data_off);
 	rxm->nb_segs = 1; /* cq_desc->num_sg_elems */
 	rxm->pkt_len = cq_desc->len;
-	rxm->port = rxq->lif->port_id;
+	rxm->port = rxq->qcq.lif->port_id;
 
 	left = cq_desc->len;
 
@@ -909,13 +915,11 @@ ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
 }
 
 static __rte_always_inline int
-ionic_rx_fill(struct ionic_qcq *rxq, uint32_t len)
+ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len)
 {
-	struct ionic_queue *q = &rxq->q;
-	struct ionic_rxq_desc *desc_base = q->base;
-	struct ionic_rxq_sg_desc *sg_desc_base = q->sg_base;
-	struct ionic_rxq_desc *desc;
-	struct ionic_rxq_sg_desc *sg_desc;
+	struct ionic_queue *q = &rxq->qcq.q;
+	struct ionic_rxq_desc *desc, *desc_base = q->base;
+	struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base;
 	struct ionic_rxq_sg_elem *elem;
 	rte_iova_t dma_addr;
 	uint32_t i, j, nsegs, buf_size, size;
@@ -990,7 +994,7 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 {
 	uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 	uint8_t *rx_queue_state = eth_dev->data->rx_queue_state;
-	struct ionic_qcq *rxq;
+	struct ionic_rx_qcq *rxq;
 	int err;
 
 	if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) {
@@ -1002,14 +1006,14 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 	rxq = eth_dev->data->rx_queues[rx_queue_id];
 
 	IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs (size: %u)",
-		rx_queue_id, rxq->q.num_descs, frame_size);
+		rx_queue_id, rxq->qcq.q.num_descs, frame_size);
 
 	if (!(rxq->flags & IONIC_QCQ_F_INITED)) {
 		err = ionic_lif_rxq_init(rxq);
 		if (err)
 			return err;
 	} else {
-		ionic_qcq_enable(rxq);
+		ionic_qcq_enable(&rxq->qcq);
 	}
 
 	/* Allocate buffers for descriptor rings */
@@ -1025,13 +1029,12 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 }
 
 static __rte_always_inline void
-ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do,
+ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
 		void *service_cb_arg)
 {
-	struct ionic_cq *cq = &rxq->cq;
-	struct ionic_queue *q = &rxq->q;
-	struct ionic_rxq_comp *cq_desc_base = cq->base;
-	struct ionic_rxq_comp *cq_desc;
+	struct ionic_cq *cq = &rxq->qcq.cq;
+	struct ionic_queue *q = &rxq->qcq.q;
+	struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base;
 	bool more;
 	uint32_t curr_q_tail_idx, curr_cq_tail_idx;
 	uint32_t work_done = 0;
@@ -1080,7 +1083,7 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do,
 int __rte_cold
 ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 {
-	struct ionic_qcq *rxq;
+	struct ionic_rx_qcq *rxq;
 
 	IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id);
 
@@ -1089,7 +1092,7 @@ ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 	eth_dev->data->rx_queue_state[rx_queue_id] =
 		RTE_ETH_QUEUE_STATE_STOPPED;
 
-	ionic_qcq_disable(rxq);
+	ionic_qcq_disable(&rxq->qcq);
 
 	/* Flush */
 	ionic_rxq_service(rxq, -1, NULL);
@@ -1101,9 +1104,9 @@ uint16_t
 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		uint16_t nb_pkts)
 {
-	struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue;
+	struct ionic_rx_qcq *rxq = rx_queue;
 	uint32_t frame_size =
-		rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
+		rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 	struct ionic_rx_service service_cb_arg;
 
 	service_cb_arg.rx_pkts = rx_pkts;