net/mlx5: fix memory region cache last index

[dpdk.git] / drivers / net / mlx5 / mlx5_rxtx.h

diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index eb7a3db..7297f01 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -200,7 +200,7 @@ struct mlx5_txq_data {
        uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
        uint16_t mr_cache_idx; /* Index of last hit entry. */
        uint32_t qp_num_8s; /* QP number shifted by 8. */
-       uint32_t flags; /* Flags for Tx Queue. */
+       uint64_t offloads; /* Offloads for Tx Queue. */
        volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */
        volatile void *wqes; /* Work queue (use volatile to write into). */
        volatile uint32_t *qp_db; /* Work queue doorbell. */
@@ -252,6 +252,7 @@ int mlx5_priv_rxq_ibv_releasable(struct priv *, struct mlx5_rxq_ibv *);
 int mlx5_priv_rxq_ibv_verify(struct priv *);
 struct mlx5_rxq_ctrl *mlx5_priv_rxq_new(struct priv *, uint16_t,
                                        uint16_t, unsigned int,
+                                       const struct rte_eth_rxconf *,
                                        struct rte_mempool *);
 struct mlx5_rxq_ctrl *mlx5_priv_rxq_get(struct priv *, uint16_t);
 int mlx5_priv_rxq_release(struct priv *, uint16_t);
@@ -272,6 +273,8 @@ struct mlx5_hrxq *mlx5_priv_hrxq_get(struct priv *, uint8_t *, uint8_t,
                                     uint64_t, uint16_t [], uint16_t);
 int mlx5_priv_hrxq_release(struct priv *, struct mlx5_hrxq *);
 int mlx5_priv_hrxq_ibv_verify(struct priv *);
+uint64_t mlx5_priv_get_rx_port_offloads(struct priv *);
+uint64_t mlx5_priv_get_rx_queue_offloads(struct priv *);
 
 /* mlx5_txq.c */
 
@@ -292,6 +295,7 @@ int mlx5_priv_txq_release(struct priv *, uint16_t);
 int mlx5_priv_txq_releasable(struct priv *, uint16_t);
 int mlx5_priv_txq_verify(struct priv *);
 void txq_alloc_elts(struct mlx5_txq_ctrl *);
+uint64_t mlx5_priv_get_tx_port_offloads(struct priv *);
 
 /* mlx5_rxtx.c */
 
@@ -309,8 +313,8 @@ int mlx5_rx_descriptor_status(void *, uint16_t);
 int mlx5_tx_descriptor_status(void *, uint16_t);
 
 /* Vectorized version of mlx5_rxtx.c */
-int priv_check_raw_vec_tx_support(struct priv *);
-int priv_check_vec_tx_support(struct priv *);
+int priv_check_raw_vec_tx_support(struct priv *, struct rte_eth_dev *);
+int priv_check_vec_tx_support(struct priv *, struct rte_eth_dev *);
 int rxq_check_vec_support(struct mlx5_rxq_data *);
 int priv_check_vec_rx_support(struct priv *);
 uint16_t mlx5_tx_burst_raw_vec(void *, struct rte_mbuf **, uint16_t);
@@ -544,27 +548,25 @@ static __rte_always_inline uint32_t
 mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
 {
        uint16_t i = txq->mr_cache_idx;
-       uintptr_t addr = rte_pktmbuf_mtod_offset(mb, uintptr_t, DATA_LEN(mb));
+       uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
        struct mlx5_mr *mr;
 
        assert(i < RTE_DIM(txq->mp2mr));
        if (likely(txq->mp2mr[i]->start <= addr && txq->mp2mr[i]->end >= addr))
                return txq->mp2mr[i]->lkey;
        for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
-               if (unlikely(txq->mp2mr[i]->mr == NULL)) {
+               if (unlikely(txq->mp2mr[i] == NULL ||
+                   txq->mp2mr[i]->mr == NULL)) {
                        /* Unknown MP, add a new MR for it. */
                        break;
                }
                if (txq->mp2mr[i]->start <= addr &&
                    txq->mp2mr[i]->end >= addr) {
                        assert(txq->mp2mr[i]->lkey != (uint32_t)-1);
-                       assert(rte_cpu_to_be_32(txq->mp2mr[i]->mr->lkey) ==
-                              txq->mp2mr[i]->lkey);
                        txq->mr_cache_idx = i;
                        return txq->mp2mr[i]->lkey;
                }
        }
-       txq->mr_cache_idx = 0;
        mr = mlx5_txq_mp2mr_reg(txq, mlx5_tx_mb2mp(mb), i);
        /*
         * Request the reference to use in this queue, the original one is
@@ -572,7 +574,13 @@ mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
         */
        if (mr) {
                rte_atomic32_inc(&mr->refcnt);
+               txq->mr_cache_idx = i >= RTE_DIM(txq->mp2mr) ? i - 1 : i;
                return mr->lkey;
+       } else {
+               struct rte_mempool *mp = mlx5_tx_mb2mp(mb);
+
+               WARN("Failed to register mempool 0x%p(%s)",
+                     (void *)mp, mp->name);
        }
        return (uint32_t)-1;
 }
@@ -594,7 +602,7 @@ mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe,
        uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
        volatile uint64_t *src = ((volatile uint64_t *)wqe);
 
-       rte_io_wmb();
+       rte_cio_wmb();
        *txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci);
        /* Ensure ordering between DB record and BF copy. */
        rte_wmb();