From be39f75cd47f590e801912c085f895d6f8143ec1 Mon Sep 17 00:00:00 2001 From: Andrew Boyer Date: Tue, 16 Feb 2021 12:35:33 -0800 Subject: [PATCH] net/ionic: split up queue-completion queue structure Create a unique Q-CQ struct for adminq, notifyq, rxq, and txq to reduce the size of each object. Minimize the size of each field to squeeze into as few cachelines as possible. Signed-off-by: Andrew Boyer --- drivers/net/ionic/ionic_lif.c | 153 ++++++++++++++++++--------------- drivers/net/ionic/ionic_lif.h | 62 +++++++++---- drivers/net/ionic/ionic_main.c | 4 +- drivers/net/ionic/ionic_rxtx.c | 129 +++++++++++++-------------- 4 files changed, 195 insertions(+), 153 deletions(-) diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index fcd6fca9a3..87579a09e5 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -76,13 +76,13 @@ ionic_lif_stop(struct ionic_lif *lif) lif->state &= ~IONIC_LIF_F_UP; for (i = 0; i < lif->nrxqcqs; i++) { - struct ionic_qcq *rxq = lif->rxqcqs[i]; + struct ionic_rx_qcq *rxq = lif->rxqcqs[i]; if (rxq->flags & IONIC_QCQ_F_INITED) (void)ionic_dev_rx_queue_stop(lif->eth_dev, i); } for (i = 0; i < lif->ntxqcqs; i++) { - struct ionic_qcq *txq = lif->txqcqs[i]; + struct ionic_tx_qcq *txq = lif->txqcqs[i]; if (txq->flags & IONIC_QCQ_F_INITED) (void)ionic_dev_tx_queue_stop(lif->eth_dev, i); } @@ -131,7 +131,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats ls->rx_bcast_bytes; for (i = 0; i < lif->nrxqcqs; i++) { - struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx; + struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats; stats->imissed += rx_stats->no_cb_arg + rx_stats->bad_cq_status + @@ -152,7 +152,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats ls->rx_desc_data_error; for (i = 0; i < num_rx_q_counters; i++) { - struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx; + struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats; stats->q_ipackets[i] = rx_stats->packets; stats->q_ibytes[i] = rx_stats->bytes; stats->q_errors[i] = @@ -173,7 +173,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats ls->tx_bcast_bytes; for (i = 0; i < lif->ntxqcqs; i++) { - struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx; + struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats; stats->oerrors += tx_stats->drop; } @@ -189,7 +189,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats ls->tx_desc_data_error; for (i = 0; i < num_tx_q_counters; i++) { - struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx; + struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats; stats->q_opackets[i] = tx_stats->packets; stats->q_obytes[i] = tx_stats->bytes; } @@ -217,9 +217,9 @@ ionic_lif_reset_stats(struct ionic_lif *lif) uint32_t i; for (i = 0; i < lif->nrxqcqs; i++) { - memset(&lif->rxqcqs[i]->stats.rx, 0, + memset(&lif->rxqcqs[i]->stats, 0, sizeof(struct ionic_rx_stats)); - memset(&lif->txqcqs[i]->stats.tx, 0, + memset(&lif->txqcqs[i]->stats, 0, sizeof(struct ionic_tx_stats)); } @@ -587,6 +587,7 @@ ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr) static int ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, + size_t struct_size, uint32_t index, const char *type_name, uint16_t flags, @@ -625,16 +626,17 @@ ionic_qcq_alloc(struct ionic_lif *lif, total_size += PAGE_SIZE; } - new = rte_zmalloc("ionic", sizeof(*new), 0); + new = rte_zmalloc("ionic", struct_size, 0); if (!new) { IONIC_PRINT(ERR, "Cannot allocate queue structure"); return -ENOMEM; } new->lif = lif; - new->flags = flags; - new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0); + new->q.info = rte_calloc_socket("ionic", + num_descs, sizeof(void *), + PAGE_SIZE, socket_id); if (!new->q.info) { IONIC_PRINT(ERR, "Cannot allocate queue info"); err = -ENOMEM; @@ -667,7 +669,6 @@ ionic_qcq_alloc(struct ionic_lif *lif, new->base = new->base_z->addr; new->base_pa = new->base_z->iova; - new->total_size = total_size; q_base = new->base; q_base_pa = new->base_pa; @@ -720,15 +721,17 @@ ionic_qcq_free(struct ionic_qcq *qcq) } int -ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs, - struct ionic_qcq **qcq) +ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, + uint16_t nrxq_descs, struct ionic_rx_qcq **rxq_out) { - uint32_t flags; - int err = -ENOMEM; + struct ionic_rx_qcq *rxq; + uint16_t flags; + int err; flags = IONIC_QCQ_F_SG; err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, + sizeof(struct ionic_rx_qcq), index, "rx", flags, @@ -736,25 +739,30 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs, sizeof(struct ionic_rxq_desc), sizeof(struct ionic_rxq_comp), sizeof(struct ionic_rxq_sg_desc), - &lif->rxqcqs[index]); + (struct ionic_qcq **)&rxq); if (err) return err; - *qcq = lif->rxqcqs[index]; + rxq->flags = flags; + + lif->rxqcqs[index] = rxq; + *rxq_out = rxq; return 0; } int -ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, - struct ionic_qcq **qcq) +ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, + uint16_t ntxq_descs, struct ionic_tx_qcq **txq_out) { - uint32_t flags; - int err = -ENOMEM; + struct ionic_tx_qcq *txq; + uint16_t flags; + int err; flags = IONIC_QCQ_F_SG; err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, + sizeof(struct ionic_tx_qcq), index, "tx", flags, @@ -762,11 +770,14 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, sizeof(struct ionic_txq_desc), sizeof(struct ionic_txq_comp), sizeof(struct ionic_txq_sg_desc_v1), - &lif->txqcqs[index]); + (struct ionic_qcq **)&txq); if (err) return err; - *qcq = lif->txqcqs[index]; + txq->flags = flags; + + lif->txqcqs[index] = txq; + *txq_out = txq; return 0; } @@ -774,12 +785,12 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, static int ionic_admin_qcq_alloc(struct ionic_lif *lif) { - uint32_t flags; - int err = -ENOMEM; + uint16_t flags = 0; + int err; - flags = 0; err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, + sizeof(struct ionic_admin_qcq), 0, "admin", flags, @@ -787,7 +798,7 @@ ionic_admin_qcq_alloc(struct ionic_lif *lif) sizeof(struct ionic_admin_cmd), sizeof(struct ionic_admin_comp), 0, - &lif->adminqcq); + (struct ionic_qcq **)&lif->adminqcq); if (err) return err; @@ -797,13 +808,14 @@ ionic_admin_qcq_alloc(struct ionic_lif *lif) static int ionic_notify_qcq_alloc(struct ionic_lif *lif) { - struct ionic_qcq *nqcq; + struct ionic_notify_qcq *nqcq; struct ionic_dev *idev = &lif->adapter->idev; - uint32_t flags = 0; - int err = -ENOMEM; + uint16_t flags = 0; + int err; err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, + sizeof(struct ionic_notify_qcq), 0, "notify", flags, @@ -811,13 +823,13 @@ ionic_notify_qcq_alloc(struct ionic_lif *lif) sizeof(struct ionic_notifyq_cmd), sizeof(union ionic_notifyq_comp), 0, - &nqcq); + (struct ionic_qcq **)&nqcq); if (err) return err; err = ionic_intr_alloc(lif, &nqcq->intr); if (err) { - ionic_qcq_free(nqcq); + ionic_qcq_free(&nqcq->qcq); return err; } @@ -1002,12 +1014,12 @@ void ionic_lif_free(struct ionic_lif *lif) { if (lif->notifyqcq) { - ionic_qcq_free(lif->notifyqcq); + ionic_qcq_free(&lif->notifyqcq->qcq); lif->notifyqcq = NULL; } if (lif->adminqcq) { - ionic_qcq_free(lif->adminqcq); + ionic_qcq_free(&lif->adminqcq->qcq); lif->adminqcq = NULL; } @@ -1137,28 +1149,28 @@ ionic_lif_rss_teardown(struct ionic_lif *lif) } } -static void -ionic_lif_qcq_deinit(struct ionic_qcq *qcq) +void +ionic_lif_txq_deinit(struct ionic_tx_qcq *txq) { - qcq->flags &= ~IONIC_QCQ_F_INITED; + txq->flags &= ~IONIC_QCQ_F_INITED; } void -ionic_lif_txq_deinit(struct ionic_qcq *qcq) +ionic_lif_rxq_deinit(struct ionic_rx_qcq *rxq) { - ionic_lif_qcq_deinit(qcq); + rxq->flags &= ~IONIC_QCQ_F_INITED; } -void -ionic_lif_rxq_deinit(struct ionic_qcq *qcq) +static void +ionic_lif_adminq_deinit(struct ionic_lif *lif) { - ionic_lif_qcq_deinit(qcq); + lif->adminqcq->flags &= ~IONIC_QCQ_F_INITED; } static void ionic_lif_notifyq_deinit(struct ionic_lif *lif) { - struct ionic_qcq *nqcq = lif->notifyqcq; + struct ionic_notify_qcq *nqcq = lif->notifyqcq; struct ionic_dev *idev = &lif->adapter->idev; if (!(nqcq->flags & IONIC_QCQ_F_INITED)) @@ -1283,26 +1295,27 @@ int ionic_notifyq_handler(struct ionic_lif *lif, int budget) { struct ionic_dev *idev = &lif->adapter->idev; - struct ionic_qcq *qcq = lif->notifyqcq; + struct ionic_notify_qcq *nqcq = lif->notifyqcq; uint32_t work_done; - if (!(qcq->flags & IONIC_QCQ_F_INITED)) { + if (!(nqcq->flags & IONIC_QCQ_F_INITED)) { IONIC_PRINT(DEBUG, "Notifyq not yet initialized"); return -1; } - ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, + ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index, IONIC_INTR_MASK_SET); - work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif); + work_done = ionic_qcq_service(&nqcq->qcq, budget, + ionic_notifyq_cb, lif); if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED) ionic_link_status_check(lif); - ionic_intr_credits(idev->intr_ctrl, qcq->intr.index, + ionic_intr_credits(idev->intr_ctrl, nqcq->intr.index, work_done, IONIC_INTR_CRED_RESET_COALESCE); - ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, + ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index, IONIC_INTR_MASK_CLEAR); return 0; @@ -1312,12 +1325,12 @@ static int ionic_lif_adminq_init(struct ionic_lif *lif) { struct ionic_dev *idev = &lif->adapter->idev; - struct ionic_qcq *qcq = lif->adminqcq; - struct ionic_queue *q = &qcq->q; + struct ionic_admin_qcq *aqcq = lif->adminqcq; + struct ionic_queue *q = &aqcq->qcq.q; struct ionic_q_init_comp comp; int err; - ionic_dev_cmd_adminq_init(idev, qcq); + ionic_dev_cmd_adminq_init(idev, &aqcq->qcq); err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); if (err) return err; @@ -1332,7 +1345,7 @@ ionic_lif_adminq_init(struct ionic_lif *lif) IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index); IONIC_PRINT(DEBUG, "adminq->db %p", q->db); - qcq->flags |= IONIC_QCQ_F_INITED; + aqcq->flags |= IONIC_QCQ_F_INITED; return 0; } @@ -1341,8 +1354,8 @@ static int ionic_lif_notifyq_init(struct ionic_lif *lif) { struct ionic_dev *idev = &lif->adapter->idev; - struct ionic_qcq *qcq = lif->notifyqcq; - struct ionic_queue *q = &qcq->q; + struct ionic_notify_qcq *nqcq = lif->notifyqcq; + struct ionic_queue *q = &nqcq->qcq.q; int err; struct ionic_admin_ctx ctx = { @@ -1352,7 +1365,7 @@ ionic_lif_notifyq_init(struct ionic_lif *lif) .type = q->type, .ver = lif->qtype_info[q->type].version, .index = rte_cpu_to_le_32(q->index), - .intr_index = rte_cpu_to_le_16(qcq->intr.index), + .intr_index = rte_cpu_to_le_16(nqcq->intr.index), .flags = rte_cpu_to_le_16(IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA), .ring_size = rte_log2_u32(q->num_descs), @@ -1378,10 +1391,10 @@ ionic_lif_notifyq_init(struct ionic_lif *lif) IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index); IONIC_PRINT(DEBUG, "notifyq->db %p", q->db); - ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, + ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index, IONIC_INTR_MASK_CLEAR); - qcq->flags |= IONIC_QCQ_F_INITED; + nqcq->flags |= IONIC_QCQ_F_INITED; return 0; } @@ -1445,8 +1458,9 @@ ionic_lif_set_features(struct ionic_lif *lif) } int -ionic_lif_txq_init(struct ionic_qcq *qcq) +ionic_lif_txq_init(struct ionic_tx_qcq *txq) { + struct ionic_qcq *qcq = &txq->qcq; struct ionic_queue *q = &qcq->q; struct ionic_lif *lif = qcq->lif; struct ionic_cq *cq = &qcq->cq; @@ -1474,7 +1488,7 @@ ionic_lif_txq_init(struct ionic_qcq *qcq) ctx.cmd.q_init.ring_size); IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver); - err = ionic_adminq_post_wait(qcq->lif, &ctx); + err = ionic_adminq_post_wait(lif, &ctx); if (err) return err; @@ -1486,14 +1500,15 @@ ionic_lif_txq_init(struct ionic_qcq *qcq) IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index); IONIC_PRINT(DEBUG, "txq->db %p", q->db); - qcq->flags |= IONIC_QCQ_F_INITED; + txq->flags |= IONIC_QCQ_F_INITED; return 0; } int -ionic_lif_rxq_init(struct ionic_qcq *qcq) +ionic_lif_rxq_init(struct ionic_rx_qcq *rxq) { + struct ionic_qcq *qcq = &rxq->qcq; struct ionic_queue *q = &qcq->q; struct ionic_lif *lif = qcq->lif; struct ionic_cq *cq = &qcq->cq; @@ -1521,7 +1536,7 @@ ionic_lif_rxq_init(struct ionic_qcq *qcq) ctx.cmd.q_init.ring_size); IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver); - err = ionic_adminq_post_wait(qcq->lif, &ctx); + err = ionic_adminq_post_wait(lif, &ctx); if (err) return err; @@ -1529,7 +1544,7 @@ ionic_lif_rxq_init(struct ionic_qcq *qcq) q->hw_index = rte_le_to_cpu_32(ctx.comp.q_init.hw_index); q->db = ionic_db_map(lif, q); - qcq->flags |= IONIC_QCQ_F_INITED; + rxq->flags |= IONIC_QCQ_F_INITED; IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type); IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index); @@ -1634,7 +1649,7 @@ err_out_notifyq_deinit: ionic_lif_notifyq_deinit(lif); err_out_adminq_deinit: - ionic_lif_qcq_deinit(lif->adminqcq); + ionic_lif_adminq_deinit(lif); return err; } @@ -1648,7 +1663,7 @@ ionic_lif_deinit(struct ionic_lif *lif) ionic_rx_filters_deinit(lif); ionic_lif_rss_teardown(lif); ionic_lif_notifyq_deinit(lif); - ionic_lif_qcq_deinit(lif->adminqcq); + ionic_lif_adminq_deinit(lif); lif->state &= ~IONIC_LIF_F_INITED; } @@ -1788,7 +1803,7 @@ ionic_lif_start(struct ionic_lif *lif) lif->nrxqcqs, lif->ntxqcqs, lif->port_id); for (i = 0; i < lif->nrxqcqs; i++) { - struct ionic_qcq *rxq = lif->rxqcqs[i]; + struct ionic_rx_qcq *rxq = lif->rxqcqs[i]; if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) { err = ionic_dev_rx_queue_start(lif->eth_dev, i); @@ -1798,7 +1813,7 @@ ionic_lif_start(struct ionic_lif *lif) } for (i = 0; i < lif->ntxqcqs; i++) { - struct ionic_qcq *txq = lif->txqcqs[i]; + struct ionic_tx_qcq *txq = lif->txqcqs[i]; if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) { err = ionic_dev_tx_queue_start(lif->eth_dev, i); diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index c850a9c08d..a8243ebf21 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -58,23 +58,47 @@ struct ionic_qcq { struct ionic_queue q; /**< Queue */ struct ionic_cq cq; /**< Completion Queue */ struct ionic_lif *lif; /**< LIF */ - struct rte_mempool *mb_pool; /**< mbuf pool to populate the RX ring */ - union { - struct ionic_tx_stats tx; - struct ionic_rx_stats rx; - } stats; const struct rte_memzone *base_z; void *base; rte_iova_t base_pa; - uint32_t total_size; - uint32_t flags; +}; + +struct ionic_admin_qcq { + struct ionic_qcq qcq; + uint16_t flags; +}; + +struct ionic_notify_qcq { + struct ionic_qcq qcq; + uint16_t flags; + struct ionic_intr_info intr; }; +struct ionic_rx_qcq { + /* cacheline0, cacheline1 */ + struct ionic_qcq qcq; + + /* cacheline2 */ + struct rte_mempool *mb_pool; + uint16_t flags; + + /* cacheline3 (inside stats) */ + struct ionic_rx_stats stats; +}; + +struct ionic_tx_qcq { + /* cacheline0, cacheline1 */ + struct ionic_qcq qcq; + + /* cacheline2 */ + uint16_t flags; + + struct ionic_tx_stats stats; +}; + #define IONIC_Q_TO_QCQ(_q) container_of(_q, struct ionic_qcq, q) #define IONIC_CQ_TO_QCQ(_cq) container_of(_cq, struct ionic_qcq, cq) -#define IONIC_Q_TO_TX_STATS(q) (&IONIC_Q_TO_QCQ(q)->stats.tx) -#define IONIC_Q_TO_RX_STATS(q) (&IONIC_Q_TO_QCQ(q)->stats.rx) struct ionic_qtype_info { uint8_t version; @@ -104,10 +128,10 @@ struct ionic_lif { uint32_t nrxqcqs; rte_spinlock_t adminq_lock; rte_spinlock_t adminq_service_lock; - struct ionic_qcq *adminqcq; - struct ionic_qcq *notifyqcq; - struct ionic_qcq **txqcqs; - struct ionic_qcq **rxqcqs; + struct ionic_admin_qcq *adminqcq; + struct ionic_notify_qcq *notifyqcq; + struct ionic_tx_qcq **txqcqs; + struct ionic_rx_qcq **rxqcqs; struct ionic_rx_filters rx_filters; struct ionic_doorbell __iomem *kern_dbpage; uint64_t last_eid; @@ -173,19 +197,19 @@ int ionic_dev_allmulticast_enable(struct rte_eth_dev *dev); int ionic_dev_allmulticast_disable(struct rte_eth_dev *dev); int ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, - uint16_t nrxq_descs, struct ionic_qcq **qcq); + uint16_t nrxq_descs, struct ionic_rx_qcq **qcq_out); int ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, - uint16_t ntxq_descs, struct ionic_qcq **qcq); + uint16_t ntxq_descs, struct ionic_tx_qcq **qcq_out); void ionic_qcq_free(struct ionic_qcq *qcq); int ionic_qcq_enable(struct ionic_qcq *qcq); int ionic_qcq_disable(struct ionic_qcq *qcq); -int ionic_lif_rxq_init(struct ionic_qcq *qcq); -void ionic_lif_rxq_deinit(struct ionic_qcq *qcq); +int ionic_lif_rxq_init(struct ionic_rx_qcq *rxq); +void ionic_lif_rxq_deinit(struct ionic_rx_qcq *rxq); -int ionic_lif_txq_init(struct ionic_qcq *qcq); -void ionic_lif_txq_deinit(struct ionic_qcq *qcq); +int ionic_lif_txq_init(struct ionic_tx_qcq *txq); +void ionic_lif_txq_deinit(struct ionic_tx_qcq *txq); int ionic_lif_rss_config(struct ionic_lif *lif, const uint16_t types, const uint8_t *key, const uint32_t *indir); diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c index 167a97dcf4..0d2e02fdd0 100644 --- a/drivers/net/ionic/ionic_main.c +++ b/drivers/net/ionic/ionic_main.c @@ -194,7 +194,7 @@ ionic_adminq_service(struct ionic_cq *cq, uint16_t cq_desc_index, static int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) { - struct ionic_queue *q = &lif->adminqcq->q; + struct ionic_queue *q = &lif->adminqcq->qcq.q; struct ionic_admin_cmd *q_desc_base = q->base; struct ionic_admin_cmd *q_desc; int err = 0; @@ -234,7 +234,7 @@ ionic_adminq_wait_for_completion(struct ionic_lif *lif, */ rte_spinlock_lock(&lif->adminq_service_lock); - ionic_qcq_service(lif->adminqcq, budget, + ionic_qcq_service(&lif->adminqcq->qcq, budget, ionic_adminq_service, NULL); rte_spinlock_unlock(&lif->adminq_service_lock); diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index 6ecb500b9e..5f836f0134 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -59,8 +59,8 @@ void ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_txq_info *qinfo) { - struct ionic_qcq *txq = dev->data->tx_queues[queue_id]; - struct ionic_queue *q = &txq->q; + struct ionic_tx_qcq *txq = dev->data->tx_queues[queue_id]; + struct ionic_queue *q = &txq->qcq.q; qinfo->nb_desc = q->num_descs; qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads; @@ -68,10 +68,10 @@ ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, } static __rte_always_inline void -ionic_tx_flush(struct ionic_qcq *txq) +ionic_tx_flush(struct ionic_tx_qcq *txq) { - struct ionic_cq *cq = &txq->cq; - struct ionic_queue *q = &txq->q; + struct ionic_cq *cq = &txq->qcq.cq; + struct ionic_queue *q = &txq->qcq.q; struct rte_mbuf *txm, *next; struct ionic_txq_comp *cq_desc_base = cq->base; struct ionic_txq_comp *cq_desc; @@ -122,19 +122,19 @@ ionic_tx_flush(struct ionic_qcq *txq) void __rte_cold ionic_dev_tx_queue_release(void *tx_queue) { - struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue; + struct ionic_tx_qcq *txq = tx_queue; IONIC_PRINT_CALL(); ionic_lif_txq_deinit(txq); - ionic_qcq_free(txq); + ionic_qcq_free(&txq->qcq); } int __rte_cold ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) { - struct ionic_qcq *txq; + struct ionic_tx_qcq *txq; IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id); @@ -148,7 +148,7 @@ ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) * before disabling Tx queue */ - ionic_qcq_disable(txq); + ionic_qcq_disable(&txq->qcq); ionic_tx_flush(txq); @@ -161,7 +161,7 @@ ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id, const struct rte_eth_txconf *tx_conf) { struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); - struct ionic_qcq *txq; + struct ionic_tx_qcq *txq; uint64_t offloads; int err; @@ -221,7 +221,7 @@ int __rte_cold ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) { uint8_t *tx_queue_state = eth_dev->data->tx_queue_state; - struct ionic_qcq *txq; + struct ionic_tx_qcq *txq; int err; if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { @@ -233,14 +233,14 @@ ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) txq = eth_dev->data->tx_queues[tx_queue_id]; IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs", - tx_queue_id, txq->q.num_descs); + tx_queue_id, txq->qcq.q.num_descs); if (!(txq->flags & IONIC_QCQ_F_INITED)) { err = ionic_lif_txq_init(txq); if (err) return err; } else { - ionic_qcq_enable(txq); + ionic_qcq_enable(&txq->qcq); } tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; @@ -315,8 +315,9 @@ ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, } static struct ionic_txq_desc * -ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem) +ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem) { + struct ionic_queue *q = &txq->qcq.q; struct ionic_txq_desc *desc_base = q->base; struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; struct ionic_txq_desc *desc = &desc_base[q->head_idx]; @@ -327,11 +328,11 @@ ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem) } static int -ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, +ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, bool not_xmit_more) { - struct ionic_queue *q = &txq->q; - struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); + struct ionic_queue *q = &txq->qcq.q; + struct ionic_tx_stats *stats = &txq->stats; struct ionic_txq_desc *desc; struct ionic_txq_sg_elem *elem; struct rte_mbuf *txm_seg; @@ -375,7 +376,7 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, left = txm->data_len; data_iova = rte_mbuf_data_iova(txm); - desc = ionic_tx_tso_next(q, &elem); + desc = ionic_tx_tso_next(txq, &elem); start = true; /* Chop data up into desc segments */ @@ -397,7 +398,7 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, encap, vlan_tci, has_vlan, start, done && not_xmit_more); - desc = ionic_tx_tso_next(q, &elem); + desc = ionic_tx_tso_next(txq, &elem); start = false; seglen = mss; } @@ -439,7 +440,7 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, encap, vlan_tci, has_vlan, start, done && not_xmit_more); - desc = ionic_tx_tso_next(q, &elem); + desc = ionic_tx_tso_next(txq, &elem); start = false; } @@ -452,16 +453,14 @@ ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, } static __rte_always_inline int -ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm, +ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, bool not_xmit_more) { - struct ionic_queue *q = &txq->q; - struct ionic_txq_desc *desc_base = q->base; + struct ionic_queue *q = &txq->qcq.q; + struct ionic_txq_desc *desc, *desc_base = q->base; struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; - struct ionic_txq_desc *desc = &desc_base[q->head_idx]; - struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx]; - struct ionic_txq_sg_elem *elem = sg_desc->elems; - struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); + struct ionic_txq_sg_elem *elem; + struct ionic_tx_stats *stats = &txq->stats; struct rte_mbuf *txm_seg; bool encap; bool has_vlan; @@ -470,6 +469,8 @@ ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm, uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; uint8_t flags = 0; + desc = &desc_base[q->head_idx]; + if ((ol_flags & PKT_TX_IP_CKSUM) && (txq->flags & IONIC_QCQ_F_CSUM_L3)) { opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; @@ -502,6 +503,7 @@ ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm, desc->len = txm->data_len; desc->vlan_tci = txm->vlan_tci; + elem = sg_desc_base[q->head_idx].elems; txm_seg = txm->next; while (txm_seg != NULL) { elem->len = txm_seg->data_len; @@ -520,9 +522,9 @@ uint16_t ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { - struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue; - struct ionic_queue *q = &txq->q; - struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); + struct ionic_tx_qcq *txq = tx_queue; + struct ionic_queue *q = &txq->qcq.q; + struct ionic_tx_stats *stats = &txq->stats; uint32_t next_q_head_idx; uint32_t bytes_tx = 0; uint16_t nb_tx = 0; @@ -625,8 +627,8 @@ void ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_rxq_info *qinfo) { - struct ionic_qcq *rxq = dev->data->rx_queues[queue_id]; - struct ionic_queue *q = &rxq->q; + struct ionic_rx_qcq *rxq = dev->data->rx_queues[queue_id]; + struct ionic_queue *q = &rxq->qcq.q; qinfo->mp = rxq->mb_pool; qinfo->scattered_rx = dev->data->scattered_rx; @@ -636,9 +638,9 @@ ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, } static void __rte_cold -ionic_rx_empty(struct ionic_queue *q) +ionic_rx_empty(struct ionic_rx_qcq *rxq) { - struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); + struct ionic_queue *q = &rxq->qcq.q; struct rte_mbuf *mbuf; void **info; @@ -654,15 +656,18 @@ ionic_rx_empty(struct ionic_queue *q) void __rte_cold ionic_dev_rx_queue_release(void *rx_queue) { - struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue; + struct ionic_rx_qcq *rxq = rx_queue; + + if (!rxq) + return; IONIC_PRINT_CALL(); - ionic_rx_empty(&rxq->q); + ionic_rx_empty(rxq); ionic_lif_rxq_deinit(rxq); - ionic_qcq_free(rxq); + ionic_qcq_free(&rxq->qcq); } int __rte_cold @@ -674,7 +679,7 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, struct rte_mempool *mp) { struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); - struct ionic_qcq *rxq; + struct ionic_rx_qcq *rxq; uint64_t offloads; int err; @@ -713,7 +718,8 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; - err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, &rxq); + err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, + &rxq); if (err) { IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id); return -EINVAL; @@ -741,20 +747,20 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, } static __rte_always_inline void -ionic_rx_clean(struct ionic_qcq *rxq, +ionic_rx_clean(struct ionic_rx_qcq *rxq, uint32_t q_desc_index, uint32_t cq_desc_index, void *service_cb_arg) { - struct ionic_queue *q = &rxq->q; - struct ionic_cq *cq = &rxq->cq; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_cq *cq = &rxq->qcq.cq; struct ionic_rxq_comp *cq_desc_base = cq->base; struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index]; struct rte_mbuf *rxm, *rxm_seg; uint32_t max_frame_size = - rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; + rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; uint64_t pkt_flags = 0; uint32_t pkt_type; - struct ionic_rx_stats *stats = IONIC_Q_TO_RX_STATS(q); + struct ionic_rx_stats *stats = &rxq->stats; struct ionic_rx_service *recv_args = (struct ionic_rx_service *) service_cb_arg; uint32_t buf_size = (uint16_t) @@ -803,7 +809,7 @@ ionic_rx_clean(struct ionic_qcq *rxq, rte_prefetch1((char *)rxm->buf_addr + rxm->data_off); rxm->nb_segs = 1; /* cq_desc->num_sg_elems */ rxm->pkt_len = cq_desc->len; - rxm->port = rxq->lif->port_id; + rxm->port = rxq->qcq.lif->port_id; left = cq_desc->len; @@ -909,13 +915,11 @@ ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, } static __rte_always_inline int -ionic_rx_fill(struct ionic_qcq *rxq, uint32_t len) +ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) { - struct ionic_queue *q = &rxq->q; - struct ionic_rxq_desc *desc_base = q->base; - struct ionic_rxq_sg_desc *sg_desc_base = q->sg_base; - struct ionic_rxq_desc *desc; - struct ionic_rxq_sg_desc *sg_desc; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_rxq_desc *desc, *desc_base = q->base; + struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; struct ionic_rxq_sg_elem *elem; rte_iova_t dma_addr; uint32_t i, j, nsegs, buf_size, size; @@ -990,7 +994,7 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) { uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; uint8_t *rx_queue_state = eth_dev->data->rx_queue_state; - struct ionic_qcq *rxq; + struct ionic_rx_qcq *rxq; int err; if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { @@ -1002,14 +1006,14 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) rxq = eth_dev->data->rx_queues[rx_queue_id]; IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs (size: %u)", - rx_queue_id, rxq->q.num_descs, frame_size); + rx_queue_id, rxq->qcq.q.num_descs, frame_size); if (!(rxq->flags & IONIC_QCQ_F_INITED)) { err = ionic_lif_rxq_init(rxq); if (err) return err; } else { - ionic_qcq_enable(rxq); + ionic_qcq_enable(&rxq->qcq); } /* Allocate buffers for descriptor rings */ @@ -1025,13 +1029,12 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) } static __rte_always_inline void -ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, +ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do, void *service_cb_arg) { - struct ionic_cq *cq = &rxq->cq; - struct ionic_queue *q = &rxq->q; - struct ionic_rxq_comp *cq_desc_base = cq->base; - struct ionic_rxq_comp *cq_desc; + struct ionic_cq *cq = &rxq->qcq.cq; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; bool more; uint32_t curr_q_tail_idx, curr_cq_tail_idx; uint32_t work_done = 0; @@ -1080,7 +1083,7 @@ ionic_rxq_service(struct ionic_qcq *rxq, uint32_t work_to_do, int __rte_cold ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) { - struct ionic_qcq *rxq; + struct ionic_rx_qcq *rxq; IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id); @@ -1089,7 +1092,7 @@ ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; - ionic_qcq_disable(rxq); + ionic_qcq_disable(&rxq->qcq); /* Flush */ ionic_rxq_service(rxq, -1, NULL); @@ -1101,9 +1104,9 @@ uint16_t ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { - struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue; + struct ionic_rx_qcq *rxq = rx_queue; uint32_t frame_size = - rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; + rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; struct ionic_rx_service service_cb_arg; service_cb_arg.rx_pkts = rx_pkts; -- 2.20.1