} while (0)
#define B_CP_DB_REARM(cpr, raw_cons) \
rte_write32((DB_CP_REARM_FLAGS | \
- RING_CMP(((cpr)->cp_ring_struct), raw_cons)), \
+ DB_RING_IDX(&((cpr)->cp_db), raw_cons)), \
((cpr)->cp_db.doorbell))
#define B_CP_DB_ARM(cpr) rte_write32((DB_KEY_CP), \
} while (0)
#define B_CP_DIS_DB(cpr, raw_cons) \
rte_write32_relaxed((DB_CP_FLAGS | \
- RING_CMP(((cpr)->cp_ring_struct), raw_cons)), \
- ((cpr)->cp_db.doorbell))
+ DB_RING_IDX(&((cpr)->cp_db), raw_cons)), \
+ ((cpr)->cp_db.doorbell))
#define B_CP_DB(cpr, raw_cons, ring_mask) \
rte_write32((DB_CP_FLAGS | \
uint32_t db_key32;
};
bool db_64;
+ uint32_t db_ring_mask;
};
+#define DB_RING_IDX(db, idx) ((idx) & (db)->db_ring_mask)
+
struct bnxt_ring;
struct bnxt_cp_ring_info {
uint32_t cp_raw_cons;
uint32_t hw_stats_ctx_id;
struct bnxt_ring *cp_ring_struct;
- uint16_t cp_cons;
bool valid;
};
memset(txr->tx_buf_ring, 0,
txr->tx_ring_struct->ring_size *
sizeof(*txr->tx_buf_ring));
- txr->tx_prod = 0;
- txr->tx_cons = 0;
+ txr->tx_raw_prod = 0;
+ txr->tx_raw_cons = 0;
}
if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) {
bnxt_free_cp_ring(bp, cpr);
mask = rep_rxr->rx_ring_struct->ring_mask;
/* Put this mbuf on the RxQ of the Representor */
- prod_rx_buf = &rep_rxr->rx_buf_ring[rep_rxr->rx_prod & mask];
+ prod_rx_buf = &rep_rxr->rx_buf_ring[rep_rxr->rx_raw_prod & mask];
if (*prod_rx_buf == NULL) {
*prod_rx_buf = mbuf;
vfr_bp->rx_bytes[que] += mbuf->pkt_len;
vfr_bp->rx_pkts[que]++;
- rep_rxr->rx_prod++;
+ rep_rxr->rx_raw_prod++;
} else {
/* Representor Rx ring full, drop pkt */
vfr_bp->rx_drop_bytes[que] += mbuf->pkt_len;
struct bnxt_db_info *db,
uint32_t ring_type,
uint32_t map_idx,
- uint32_t fid)
+ uint32_t fid,
+ uint32_t ring_mask)
{
if (BNXT_CHIP_P5(bp)) {
if (BNXT_PF(bp))
}
db->db_64 = false;
}
+ db->db_ring_mask = ring_mask;
}
static int bnxt_alloc_cmpl_ring(struct bnxt *bp, int queue_index,
if (rc)
return rc;
- cpr->cp_cons = 0;
+ cpr->cp_raw_cons = 0;
bnxt_set_db(bp, &cpr->cp_db, ring_type, cp_ring_index,
- cp_ring->fw_ring_id);
+ cp_ring->fw_ring_id, cp_ring->ring_mask);
bnxt_db_cq(cpr);
return 0;
}
bnxt_set_db(bp, &nqr->cp_db, ring_type, ring_index,
- ring->fw_ring_id);
+ ring->fw_ring_id, ring->ring_mask);
bnxt_db_nq(nqr);
bp->rxtx_nq_ring = nqr;
if (rc)
return rc;
- rxr->rx_prod = 0;
+ rxr->rx_raw_prod = 0;
if (BNXT_HAS_RING_GRPS(bp))
bp->grp_info[queue_index].rx_fw_ring_id = ring->fw_ring_id;
- bnxt_set_db(bp, &rxr->rx_db, ring_type, queue_index, ring->fw_ring_id);
- bnxt_db_write(&rxr->rx_db, rxr->rx_prod);
+ bnxt_set_db(bp, &rxr->rx_db, ring_type, queue_index, ring->fw_ring_id,
+ ring->ring_mask);
+ bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod);
return 0;
}
if (rc)
return rc;
- rxr->ag_prod = 0;
+ rxr->ag_raw_prod = 0;
if (BNXT_HAS_RING_GRPS(bp))
bp->grp_info[queue_index].ag_fw_ring_id = ring->fw_ring_id;
- bnxt_set_db(bp, &rxr->ag_db, ring_type, map_idx, ring->fw_ring_id);
- bnxt_db_write(&rxr->ag_db, rxr->ag_prod);
+ bnxt_set_db(bp, &rxr->ag_db, ring_type, map_idx, ring->fw_ring_id,
+ ring->ring_mask);
+ bnxt_db_write(&rxr->ag_db, rxr->ag_raw_prod);
return 0;
}
rc = -ENOMEM;
goto err_out;
}
- bnxt_db_write(&rxr->rx_db, rxr->rx_prod);
- bnxt_db_write(&rxr->ag_db, rxr->ag_prod);
+ bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod);
+ bnxt_db_write(&rxr->ag_db, rxr->ag_raw_prod);
}
rxq->index = queue_index;
#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
bnxt_rx_queue_release_op(rxq);
return -ENOMEM;
}
- bnxt_db_write(&rxr->rx_db, rxr->rx_prod);
- bnxt_db_write(&rxr->ag_db, rxr->ag_prod);
+ bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod);
+ bnxt_db_write(&rxr->ag_db, rxr->ag_raw_prod);
rxq->index = i;
#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
bnxt_rxq_vec_setup(rxq);
if (rc)
goto err_out;
- bnxt_set_db(bp, &txr->tx_db, ring_type, i, ring->fw_ring_id);
+ bnxt_set_db(bp, &txr->tx_db, ring_type, i, ring->fw_ring_id,
+ ring->ring_mask);
txq->index = idx;
bnxt_hwrm_set_ring_coal(bp, &coal, cp_ring->fw_ring_id);
}
if (rc)
return rc;
- cpr->cp_cons = 0;
+ cpr->cp_raw_cons = 0;
cpr->valid = 0;
bnxt_set_db(bp, &cpr->cp_db, ring_type, 0,
- cp_ring->fw_ring_id);
+ cp_ring->fw_ring_id, cp_ring->ring_mask);
if (BNXT_HAS_NQ(bp))
bnxt_db_nq(cpr);
#include <rte_memory.h>
-#define RING_ADV(ring, idx, n) (((idx) + (n)) & (ring)->ring_mask)
-#define RING_NEXT(ring, idx) RING_ADV(ring, idx, 1)
+#define RING_ADV(idx, n) ((idx) + (n))
+#define RING_NEXT(idx) RING_ADV(idx, 1)
+#define RING_IDX(ring, idx) ((idx) & (ring)->ring_mask)
#define DB_IDX_MASK 0xffffff
#define DB_IDX_VALID (0x1 << 26)
static inline void bnxt_db_write(struct bnxt_db_info *db, uint32_t idx)
{
- if (db->db_64)
- rte_write64(db->db_key64 | idx, db->doorbell);
- else
- rte_write32(db->db_key32 | idx, db->doorbell);
+ uint32_t db_idx = DB_RING_IDX(db, idx);
+ void *doorbell = db->doorbell;
+
+ if (db->db_64) {
+ uint64_t key_idx = db->db_key64 | db_idx;
+
+ rte_write64(key_idx, doorbell);
+ } else {
+ uint32_t key_idx = db->db_key32 | db_idx;
+
+ rte_write32(key_idx, doorbell);
+ }
}
/* Ring an NQ doorbell and disable interrupts for the ring. */
static inline void bnxt_db_nq(struct bnxt_cp_ring_info *cpr)
{
+ uint32_t db_idx = DB_RING_IDX(&cpr->cp_db, cpr->cp_raw_cons);
+ uint64_t key_idx = cpr->cp_db.db_key64 | DBR_TYPE_NQ | db_idx;
+ void *doorbell = cpr->cp_db.doorbell;
+
+
if (unlikely(!cpr->cp_db.db_64))
return;
- rte_write64(cpr->cp_db.db_key64 | DBR_TYPE_NQ |
- RING_CMP(cpr->cp_ring_struct, cpr->cp_raw_cons),
- cpr->cp_db.doorbell);
+ rte_write64(key_idx, doorbell);
}
/* Ring an NQ doorbell and enable interrupts for the ring. */
static inline void bnxt_db_nq_arm(struct bnxt_cp_ring_info *cpr)
{
+ uint32_t db_idx = DB_RING_IDX(&cpr->cp_db, cpr->cp_raw_cons);
+ uint64_t key_idx = cpr->cp_db.db_key64 | DBR_TYPE_NQ_ARM | db_idx;
+ void *doorbell = cpr->cp_db.doorbell;
+
if (unlikely(!cpr->cp_db.db_64))
return;
- rte_write64(cpr->cp_db.db_key64 | DBR_TYPE_NQ_ARM |
- RING_CMP(cpr->cp_ring_struct, cpr->cp_raw_cons),
- cpr->cp_db.doorbell);
+ rte_write64(key_idx, doorbell);
}
static inline void bnxt_db_cq(struct bnxt_cp_ring_info *cpr)
{
struct bnxt_db_info *db = &cpr->cp_db;
- uint32_t idx = RING_CMP(cpr->cp_ring_struct, cpr->cp_raw_cons);
+ uint32_t idx = DB_RING_IDX(&cpr->cp_db, cpr->cp_raw_cons);
if (db->db_64) {
uint64_t key_idx = db->db_key64 | idx;
static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue *rxq,
struct bnxt_rx_ring_info *rxr,
- uint16_t prod)
+ uint16_t raw_prod)
{
- struct rx_prod_pkt_bd *rxbd = &rxr->rx_desc_ring[prod];
- struct rte_mbuf **rx_buf = &rxr->rx_buf_ring[prod];
+ uint16_t prod = RING_IDX(rxr->rx_ring_struct, raw_prod);
+ struct rx_prod_pkt_bd *rxbd;
+ struct rte_mbuf **rx_buf;
struct rte_mbuf *mbuf;
+ rxbd = &rxr->rx_desc_ring[prod];
+ rx_buf = &rxr->rx_buf_ring[prod];
mbuf = __bnxt_alloc_rx_data(rxq->mb_pool);
if (!mbuf) {
rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail);
static inline int bnxt_alloc_ag_data(struct bnxt_rx_queue *rxq,
struct bnxt_rx_ring_info *rxr,
- uint16_t prod)
+ uint16_t raw_prod)
{
- struct rx_prod_pkt_bd *rxbd = &rxr->ag_desc_ring[prod];
- struct rte_mbuf **rx_buf = &rxr->ag_buf_ring[prod];
+ uint16_t prod = RING_IDX(rxr->ag_ring_struct, raw_prod);
+ struct rx_prod_pkt_bd *rxbd;
+ struct rte_mbuf **rx_buf;
struct rte_mbuf *mbuf;
+ rxbd = &rxr->ag_desc_ring[prod];
+ rx_buf = &rxr->ag_buf_ring[prod];
if (rxbd == NULL) {
PMD_DRV_LOG(ERR, "Jumbo Frame. rxbd is NULL\n");
return -EINVAL;
static inline void bnxt_reuse_rx_mbuf(struct bnxt_rx_ring_info *rxr,
struct rte_mbuf *mbuf)
{
- uint16_t prod = RING_NEXT(rxr->rx_ring_struct, rxr->rx_prod);
+ uint16_t prod, raw_prod = RING_NEXT(rxr->rx_raw_prod);
struct rte_mbuf **prod_rx_buf;
struct rx_prod_pkt_bd *prod_bd;
+ prod = RING_IDX(rxr->rx_ring_struct, raw_prod);
prod_rx_buf = &rxr->rx_buf_ring[prod];
RTE_ASSERT(*prod_rx_buf == NULL);
prod_bd->address = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
- rxr->rx_prod = prod;
+ rxr->rx_raw_prod = raw_prod;
}
static inline
struct rte_mbuf **cons_rx_buf;
struct rte_mbuf *mbuf;
- cons_rx_buf = &rxr->rx_buf_ring[cons];
+ cons_rx_buf = &rxr->rx_buf_ring[RING_IDX(rxr->rx_ring_struct, cons)];
RTE_ASSERT(*cons_rx_buf != NULL);
mbuf = *cons_rx_buf;
*cons_rx_buf = NULL;
mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
/* recycle next mbuf */
- data_cons = RING_NEXT(rxr->rx_ring_struct, data_cons);
+ data_cons = RING_NEXT(data_cons);
bnxt_reuse_rx_mbuf(rxr, bnxt_consume_rx_buf(rxr, data_cons));
}
static int bnxt_prod_ag_mbuf(struct bnxt_rx_queue *rxq)
{
struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
- uint16_t next = RING_NEXT(rxr->ag_ring_struct, rxr->ag_prod);
+ uint16_t raw_next = RING_NEXT(rxr->ag_raw_prod);
+ uint16_t bmap_next = RING_IDX(rxr->ag_ring_struct, raw_next);
/* TODO batch allocation for better performance */
- while (rte_bitmap_get(rxr->ag_bitmap, next)) {
- if (unlikely(bnxt_alloc_ag_data(rxq, rxr, next))) {
- PMD_DRV_LOG(ERR,
- "agg mbuf alloc failed: prod=0x%x\n", next);
+ while (rte_bitmap_get(rxr->ag_bitmap, bmap_next)) {
+ if (unlikely(bnxt_alloc_ag_data(rxq, rxr, raw_next))) {
+ PMD_DRV_LOG(ERR, "agg mbuf alloc failed: prod=0x%x\n",
+ raw_next);
break;
}
- rte_bitmap_clear(rxr->ag_bitmap, next);
- rxr->ag_prod = next;
- next = RING_NEXT(rxr->ag_ring_struct, next);
+ rte_bitmap_clear(rxr->ag_bitmap, bmap_next);
+ rxr->ag_raw_prod = raw_next;
+ raw_next = RING_NEXT(raw_next);
+ bmap_next = RING_IDX(rxr->ag_ring_struct, raw_next);
}
return 0;
struct rx_pkt_cmpl *rxcmp;
struct rx_pkt_cmpl_hi *rxcmp1;
uint32_t tmp_raw_cons = *raw_cons;
- uint16_t cons, prod, cp_cons =
+ uint16_t cons, raw_prod, cp_cons =
RING_CMP(cpr->cp_ring_struct, tmp_raw_cons);
struct rte_mbuf *mbuf;
int rc = 0;
if (agg_buf && !bnxt_agg_bufs_valid(cpr, agg_buf, tmp_raw_cons))
return -EBUSY;
- prod = rxr->rx_prod;
+ raw_prod = rxr->rx_raw_prod;
cons = rxcmp->opaque;
mbuf = bnxt_consume_rx_buf(rxr, cons);
* calls in favour of a tight loop with the same function being called
* in it.
*/
- prod = RING_NEXT(rxr->rx_ring_struct, prod);
- if (bnxt_alloc_rx_data(rxq, rxr, prod)) {
- PMD_DRV_LOG(ERR, "mbuf alloc failed with prod=0x%x\n", prod);
+ raw_prod = RING_NEXT(raw_prod);
+ if (bnxt_alloc_rx_data(rxq, rxr, raw_prod)) {
+ PMD_DRV_LOG(ERR, "mbuf alloc failed with prod=0x%x\n",
+ raw_prod);
rc = -ENOMEM;
goto rx;
}
- rxr->rx_prod = prod;
+ rxr->rx_raw_prod = raw_prod;
if (BNXT_TRUFLOW_EN(bp) && (BNXT_VF_IS_TRUSTED(bp) || BNXT_PF(bp)) &&
vfr_flag) {
struct bnxt_rx_queue *rxq = rx_queue;
struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
+ uint16_t rx_raw_prod = rxr->rx_raw_prod;
+ uint16_t ag_raw_prod = rxr->ag_raw_prod;
uint32_t raw_cons = cpr->cp_raw_cons;
uint32_t cons;
int nb_rx_pkts = 0;
int nb_rep_rx_pkts = 0;
struct rx_pkt_cmpl *rxcmp;
- uint16_t prod = rxr->rx_prod;
- uint16_t ag_prod = rxr->ag_prod;
int rc = 0;
bool evt = false;
*/
while (unlikely(rxq->rxrearm_nb)) {
if (!bnxt_alloc_rx_data(rxq, rxr, rxq->rxrearm_start)) {
- rxr->rx_prod = rxq->rxrearm_start;
- bnxt_db_write(&rxr->rx_db, rxr->rx_prod);
+ rxr->rx_raw_prod = rxq->rxrearm_start;
+ bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod);
rxq->rxrearm_start++;
rxq->rxrearm_nb--;
} else {
break;
/* Post some Rx buf early in case of larger burst processing */
if (nb_rx_pkts == BNXT_RX_POST_THRESH)
- bnxt_db_write(&rxr->rx_db, rxr->rx_prod);
+ bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod);
}
cpr->cp_raw_cons = raw_cons;
goto done;
}
- if (prod != rxr->rx_prod)
- bnxt_db_write(&rxr->rx_db, rxr->rx_prod);
+ rte_compiler_barrier();
+ if (rx_raw_prod != rxr->rx_raw_prod)
+ bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod);
+ rte_compiler_barrier();
/* Ring the AGG ring DB */
- if (ag_prod != rxr->ag_prod)
- bnxt_db_write(&rxr->ag_db, rxr->ag_prod);
+ if (ag_raw_prod != rxr->ag_raw_prod)
+ bnxt_db_write(&rxr->ag_db, rxr->ag_raw_prod);
bnxt_db_cq(cpr);
/* Attempt to alloc Rx buf in case of a previous allocation failure. */
if (rc == -ENOMEM) {
- int i = RING_NEXT(rxr->rx_ring_struct, prod);
+ int i = RING_NEXT(rx_raw_prod);
int cnt = nb_rx_pkts;
- for (; cnt;
- i = RING_NEXT(rxr->rx_ring_struct, i), cnt--) {
- struct rte_mbuf **rx_buf = &rxr->rx_buf_ring[i];
+ for (; nb_rx_pkts; i = RING_NEXT(i), cnt--) {
+ struct rte_mbuf **rx_buf;
+ uint16_t rx_raw_prod = RING_IDX(rxr->rx_ring_struct, i);
+
+ rx_buf = &rxr->rx_buf_ring[rx_raw_prod];
/* Buffer already allocated for this index. */
if (*rx_buf != NULL && *rx_buf != &rxq->fake_mbuf)
/* This slot is empty. Alloc buffer for Rx */
if (!bnxt_alloc_rx_data(rxq, rxr, i)) {
- rxr->rx_prod = i;
- bnxt_db_write(&rxr->rx_db, rxr->rx_prod);
+ rxr->rx_raw_prod = i;
+ bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod);
} else {
PMD_DRV_LOG(ERR, "Alloc mbuf failed\n");
break;
{
struct bnxt_rx_ring_info *rxr;
struct bnxt_ring *ring;
- uint32_t prod, type;
+ uint32_t raw_prod, type;
unsigned int i;
uint16_t size;
ring = rxr->rx_ring_struct;
bnxt_init_rxbds(ring, type, size);
- prod = rxr->rx_prod;
+ raw_prod = rxr->rx_raw_prod;
for (i = 0; i < ring->ring_size; i++) {
if (unlikely(!rxr->rx_buf_ring[i])) {
- if (bnxt_alloc_rx_data(rxq, rxr, prod) != 0) {
+ if (bnxt_alloc_rx_data(rxq, rxr, raw_prod) != 0) {
PMD_DRV_LOG(WARNING,
"init'ed rx ring %d with %d/%d mbufs only\n",
rxq->queue_id, i, ring->ring_size);
break;
}
}
- rxr->rx_prod = prod;
- prod = RING_NEXT(rxr->rx_ring_struct, prod);
+ rxr->rx_raw_prod = raw_prod;
+ raw_prod = RING_NEXT(raw_prod);
}
/* Initialize dummy mbuf pointers for vector mode rx. */
ring = rxr->ag_ring_struct;
type = RX_PROD_AGG_BD_TYPE_RX_PROD_AGG;
bnxt_init_rxbds(ring, type, size);
- prod = rxr->ag_prod;
+ raw_prod = rxr->ag_raw_prod;
for (i = 0; i < ring->ring_size; i++) {
if (unlikely(!rxr->ag_buf_ring[i])) {
- if (bnxt_alloc_ag_data(rxq, rxr, prod) != 0) {
+ if (bnxt_alloc_ag_data(rxq, rxr, raw_prod) != 0) {
PMD_DRV_LOG(WARNING,
"init'ed AG ring %d with %d/%d mbufs only\n",
rxq->queue_id, i, ring->ring_size);
break;
}
}
- rxr->ag_prod = prod;
- prod = RING_NEXT(rxr->ag_ring_struct, prod);
+ rxr->ag_raw_prod = raw_prod;
+ raw_prod = RING_NEXT(raw_prod);
}
PMD_DRV_LOG(DEBUG, "AGG Done!\n");
};
struct bnxt_rx_ring_info {
- uint16_t rx_prod;
- uint16_t ag_prod;
+ uint16_t rx_raw_prod;
+ uint16_t ag_raw_prod;
uint16_t rx_cons; /* Needed for representor */
struct bnxt_db_info rx_db;
struct bnxt_db_info ag_db;
bnxt_tx_cmp_vec_fast(struct bnxt_tx_queue *txq, int nr_pkts)
{
struct bnxt_tx_ring_info *txr = txq->tx_ring;
- uint32_t ring_mask = txr->tx_ring_struct->ring_mask;
struct rte_mbuf **free = txq->free;
- uint16_t cons = txr->tx_cons;
+ uint16_t cons, raw_cons = txr->tx_raw_cons;
unsigned int blk = 0;
+ uint32_t ring_mask = txr->tx_ring_struct->ring_mask;
while (nr_pkts--) {
struct bnxt_sw_tx_bd *tx_buf;
+ cons = raw_cons++ & ring_mask;
tx_buf = &txr->tx_buf_ring[cons];
- cons = (cons + 1) & ring_mask;
free[blk++] = tx_buf->mbuf;
tx_buf->mbuf = NULL;
}
if (blk)
rte_mempool_put_bulk(free[0]->pool, (void **)free, blk);
- txr->tx_cons = cons;
+ txr->tx_raw_cons = raw_cons;
}
static inline void
{
struct bnxt_tx_ring_info *txr = txq->tx_ring;
struct rte_mbuf **free = txq->free;
- uint16_t cons = txr->tx_cons;
+ uint16_t cons, raw_cons = txr->tx_raw_cons;
unsigned int blk = 0;
uint32_t ring_mask = txr->tx_ring_struct->ring_mask;
struct bnxt_sw_tx_bd *tx_buf;
struct rte_mbuf *mbuf;
+ cons = raw_cons++ & ring_mask;
tx_buf = &txr->tx_buf_ring[cons];
- cons = (cons + 1) & ring_mask;
mbuf = rte_pktmbuf_prefree_seg(tx_buf->mbuf);
if (unlikely(mbuf == NULL))
continue;
if (blk)
rte_mempool_put_bulk(free[0]->pool, (void **)free, blk);
- txr->tx_cons = cons;
+ txr->tx_raw_cons = raw_cons;
}
#endif /* _BNXT_RXTX_VEC_COMMON_H_ */
out:
if (nb_rx_pkts) {
- rxr->rx_prod =
- RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts);
+ rxr->rx_raw_prod = RING_ADV(rxr->rx_raw_prod, nb_rx_pkts);
rxq->rxrearm_nb += nb_rx_pkts;
cpr->cp_raw_cons += 2 * nb_rx_pkts;
{
struct bnxt_tx_queue *txq = tx_queue;
struct bnxt_tx_ring_info *txr = txq->tx_ring;
- uint16_t prod = txr->tx_prod;
+ uint16_t tx_prod, tx_raw_prod = txr->tx_raw_prod;
struct rte_mbuf *tx_mbuf;
struct tx_bd_long *txbd = NULL;
struct bnxt_sw_tx_bd *tx_buf;
tx_mbuf = *tx_pkts++;
rte_prefetch0(tx_mbuf);
- tx_buf = &txr->tx_buf_ring[prod];
+ tx_prod = RING_IDX(txr->tx_ring_struct, tx_raw_prod);
+ tx_buf = &txr->tx_buf_ring[tx_prod];
tx_buf->mbuf = tx_mbuf;
tx_buf->nr_bds = 1;
- txbd = &txr->tx_desc_ring[prod];
+ txbd = &txr->tx_desc_ring[tx_prod];
txbd->address = tx_mbuf->buf_iova + tx_mbuf->data_off;
txbd->len = tx_mbuf->data_len;
txbd->flags_type = bnxt_xmit_flags_len(tx_mbuf->data_len,
TX_BD_FLAGS_NOCMPL);
- prod = RING_NEXT(txr->tx_ring_struct, prod);
+ tx_raw_prod = RING_NEXT(tx_raw_prod);
to_send--;
}
}
rte_compiler_barrier();
- bnxt_db_write(&txr->tx_db, prod);
+ bnxt_db_write(&txr->tx_db, tx_raw_prod);
- txr->tx_prod = prod;
+ txr->tx_raw_prod = tx_raw_prod;
return nb_pkts;
}
out:
if (nb_rx_pkts) {
- rxr->rx_prod =
- RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts);
+ rxr->rx_raw_prod = RING_ADV(rxr->rx_raw_prod, nb_rx_pkts);
rxq->rxrearm_nb += nb_rx_pkts;
cpr->cp_raw_cons += 2 * nb_rx_pkts;
uint16_t nb_pkts)
{
struct bnxt_tx_ring_info *txr = txq->tx_ring;
- uint16_t tx_prod = txr->tx_prod;
+ uint16_t tx_prod, tx_raw_prod = txr->tx_raw_prod;
struct tx_bd_long *txbd;
struct bnxt_sw_tx_bd *tx_buf;
uint16_t to_send;
+ tx_prod = RING_IDX(txr->tx_ring_struct, tx_raw_prod);
txbd = &txr->tx_desc_ring[tx_prod];
tx_buf = &txr->tx_buf_ring[tx_prod];
txbd[-1].opaque = nb_pkts;
txbd[-1].flags_type &= ~TX_BD_LONG_FLAGS_NO_CMPL;
- tx_prod = RING_ADV(txr->tx_ring_struct, tx_prod, nb_pkts);
- bnxt_db_write(&txr->tx_db, tx_prod);
+ tx_raw_prod += nb_pkts;
+ bnxt_db_write(&txr->tx_db, tx_raw_prod);
- txr->tx_prod = tx_prod;
+ txr->tx_raw_prod = tx_raw_prod;
return nb_pkts;
}
* Ensure that a ring wrap does not occur within a call to
* bnxt_xmit_fixed_burst_vec().
*/
- num = RTE_MIN(num,
- ring_size - (txr->tx_prod & (ring_size - 1)));
+ num = RTE_MIN(num, ring_size -
+ (txr->tx_raw_prod & (ring_size - 1)));
ret = bnxt_xmit_fixed_burst_vec(txq, &tx_pkts[nb_sent], num);
nb_sent += ret;
nb_pkts -= ret;
struct tx_bd_long **last_txbd)
{
struct bnxt_tx_ring_info *txr = txq->tx_ring;
+ struct bnxt_ring *ring = txr->tx_ring_struct;
uint32_t outer_tpid_bd = 0;
struct tx_bd_long *txbd;
struct tx_bd_long_hi *txbd1 = NULL;
uint32_t vlan_tag_flags;
bool long_bd = false;
unsigned short nr_bds = 0;
+ uint16_t prod;
struct rte_mbuf *m_seg;
struct bnxt_sw_tx_bd *tx_buf;
static const uint32_t lhint_arr[4] = {
/* Check non zero data_len */
RTE_VERIFY(tx_pkt->data_len);
- tx_buf = &txr->tx_buf_ring[txr->tx_prod];
+ prod = RING_IDX(ring, txr->tx_raw_prod);
+ tx_buf = &txr->tx_buf_ring[prod];
tx_buf->mbuf = tx_pkt;
tx_buf->nr_bds = nr_bds;
- txbd = &txr->tx_desc_ring[txr->tx_prod];
+ txbd = &txr->tx_desc_ring[prod];
txbd->opaque = *coal_pkts;
txbd->flags_type = nr_bds << TX_BD_LONG_FLAGS_BD_CNT_SFT;
txbd->flags_type |= TX_BD_SHORT_FLAGS_COAL_NOW;
TX_BD_LONG_CFA_META_VLAN_TPID_TPID8100;
}
- txr->tx_prod = RING_NEXT(txr->tx_ring_struct, txr->tx_prod);
+ txr->tx_raw_prod = RING_NEXT(txr->tx_raw_prod);
- txbd1 = (struct tx_bd_long_hi *)
- &txr->tx_desc_ring[txr->tx_prod];
+ prod = RING_IDX(ring, txr->tx_raw_prod);
+ txbd1 = (struct tx_bd_long_hi *)&txr->tx_desc_ring[prod];
txbd1->lflags = 0;
txbd1->cfa_meta = vlan_tag_flags;
/* Legacy tx_bd_long_hi->mss =
while (m_seg) {
/* Check non zero data_len */
RTE_VERIFY(m_seg->data_len);
- txr->tx_prod = RING_NEXT(txr->tx_ring_struct, txr->tx_prod);
- tx_buf = &txr->tx_buf_ring[txr->tx_prod];
+ txr->tx_raw_prod = RING_NEXT(txr->tx_raw_prod);
+
+ prod = RING_IDX(ring, txr->tx_raw_prod);
+ tx_buf = &txr->tx_buf_ring[prod];
tx_buf->mbuf = m_seg;
- txbd = &txr->tx_desc_ring[txr->tx_prod];
+ txbd = &txr->tx_desc_ring[prod];
txbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova(m_seg));
txbd->flags_type = TX_BD_SHORT_TYPE_TX_BD_SHORT;
txbd->len = m_seg->data_len;
txbd->flags_type |= TX_BD_LONG_FLAGS_PACKET_END;
- txr->tx_prod = RING_NEXT(txr->tx_ring_struct, txr->tx_prod);
+ txr->tx_raw_prod = RING_NEXT(txr->tx_raw_prod);
return 0;
}
static void bnxt_tx_cmp_fast(struct bnxt_tx_queue *txq, int nr_pkts)
{
struct bnxt_tx_ring_info *txr = txq->tx_ring;
+ struct bnxt_ring *ring = txr->tx_ring_struct;
struct rte_mbuf **free = txq->free;
- uint16_t cons = txr->tx_cons;
+ uint16_t raw_cons = txr->tx_raw_cons;
unsigned int blk = 0;
int i, j;
struct bnxt_sw_tx_bd *tx_buf;
unsigned short nr_bds;
- tx_buf = &txr->tx_buf_ring[cons];
+ tx_buf = &txr->tx_buf_ring[RING_IDX(ring, raw_cons)];
nr_bds = tx_buf->nr_bds;
for (j = 0; j < nr_bds; j++) {
if (tx_buf->mbuf) {
free[blk++] = tx_buf->mbuf;
tx_buf->mbuf = NULL;
}
- cons = RING_NEXT(txr->tx_ring_struct, cons);
- tx_buf = &txr->tx_buf_ring[cons];
+ raw_cons = RING_NEXT(raw_cons);
+ tx_buf = &txr->tx_buf_ring[RING_IDX(ring, raw_cons)];
}
}
if (blk)
rte_mempool_put_bulk(free[0]->pool, (void *)free, blk);
- txr->tx_cons = cons;
+ txr->tx_raw_cons = raw_cons;
}
static void bnxt_tx_cmp(struct bnxt_tx_queue *txq, int nr_pkts)
{
struct bnxt_tx_ring_info *txr = txq->tx_ring;
+ struct bnxt_ring *ring = txr->tx_ring_struct;
struct rte_mempool *pool = NULL;
struct rte_mbuf **free = txq->free;
- uint16_t cons = txr->tx_cons;
+ uint16_t raw_cons = txr->tx_raw_cons;
unsigned int blk = 0;
int i, j;
for (i = 0; i < nr_pkts; i++) {
struct rte_mbuf *mbuf;
- struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[cons];
- unsigned short nr_bds = tx_buf->nr_bds;
+ struct bnxt_sw_tx_bd *tx_buf;
+ unsigned short nr_bds;
+ tx_buf = &txr->tx_buf_ring[RING_IDX(ring, raw_cons)];
+ nr_bds = tx_buf->nr_bds;
for (j = 0; j < nr_bds; j++) {
mbuf = tx_buf->mbuf;
tx_buf->mbuf = NULL;
- cons = RING_NEXT(txr->tx_ring_struct, cons);
- tx_buf = &txr->tx_buf_ring[cons];
+ raw_cons = RING_NEXT(raw_cons);
+ tx_buf = &txr->tx_buf_ring[RING_IDX(ring, raw_cons)];
if (!mbuf) /* long_bd's tx_buf ? */
continue;
if (blk)
rte_mempool_put_bulk(pool, (void *)free, blk);
- txr->tx_cons = cons;
+ txr->tx_raw_cons = raw_cons;
}
static int bnxt_handle_tx_cp(struct bnxt_tx_queue *txq)
if (likely(nb_tx_pkts)) {
/* Request a completion on the last packet */
last_txbd->flags_type &= ~TX_BD_LONG_FLAGS_NO_CMPL;
- bnxt_db_write(&txq->tx_ring->tx_db, txq->tx_ring->tx_prod);
+ bnxt_db_write(&txq->tx_ring->tx_db, txq->tx_ring->tx_raw_prod);
}
return nb_tx_pkts;
#define BNXT_MIN_PKT_SIZE 52
struct bnxt_tx_ring_info {
- uint16_t tx_prod;
- uint16_t tx_cons;
+ uint16_t tx_raw_prod;
+ uint16_t tx_raw_cons;
struct bnxt_db_info tx_db;
struct tx_bd_long *tx_desc_ring;
static inline uint32_t bnxt_tx_bds_in_hw(struct bnxt_tx_queue *txq)
{
- return ((txq->tx_ring->tx_prod - txq->tx_ring->tx_cons) &
+ return ((txq->tx_ring->tx_raw_prod - txq->tx_ring->tx_raw_cons) &
txq->tx_ring->tx_ring_struct->ring_mask);
}
git.droids-corp.org / dpdk.git / commitdiff