net/mlx5: fix split buffer Rx

Routine to lookup LKey on Rx was assuming that the mbuf address
always belongs to a single mempool: the one associated with an RxQ
or the MPRQ mempool. This assumption is false for split buffers case.
A wrong LKey was looked up, resulting in completion errors.
Modify lookup routines to lookup LKey in the mbuf->pool
for non-MPRQ cases both on Rx datapath and on queue initialization.

Fixes: fec28ca0e3a9 ("net/mlx5: support mempool registration")
Signed-off-by: Dmitry Kozlyuk <dkozlyuk@nvidia.com>
Reviewed-by: Matan Azrad <matan@nvidia.com>
Reviewed-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>

diff --git a/drivers/net/mlx5/mlx5_rx.c b/drivers/net/mlx5/mlx5_rx.c
index 4d85f64accdd29a21c04cff5b09e3dbb83818681..e8215f738150360ca276ac1921987066d92b69ed 100644 (file)
--- a/drivers/net/mlx5/mlx5_rx.c
+++ b/drivers/net/mlx5/mlx5_rx.c
@@ -347,6 +347,7 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
                volatile struct mlx5_wqe_data_seg *scat;
                uintptr_t addr;
                uint32_t byte_count;
+               uint32_t lkey;
 
                if (mlx5_rxq_mprq_enabled(rxq)) {
                        struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[i];
@@ -357,6 +358,7 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
                                                         1 << rxq->strd_num_n);
                        byte_count = (1 << rxq->strd_sz_n) *
                                        (1 << rxq->strd_num_n);
+                       lkey = mlx5_rx_addr2mr(rxq, addr);
                } else {
                        struct rte_mbuf *buf = (*rxq->elts)[i];
 
@@ -364,13 +366,14 @@ mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
                                        rxq->wqes)[i];
                        addr = rte_pktmbuf_mtod(buf, uintptr_t);
                        byte_count = DATA_LEN(buf);
+                       lkey = mlx5_rx_mb2mr(rxq, buf);
                }
                /* scat->addr must be able to store a pointer. */
                MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t));
                *scat = (struct mlx5_wqe_data_seg){
                        .addr = rte_cpu_to_be_64(addr),
                        .byte_count = rte_cpu_to_be_32(byte_count),
-                       .lkey = mlx5_rx_addr2mr(rxq, addr),
+                       .lkey = lkey,
                };
        }
        rxq->consumed_strd = 0;

diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 19d194ce12a4bcac6d9131a5bd1ca19619abb6f0..298d1b1f3265d72950fd9b7a77832c2068aa883a 100644 (file)
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -291,7 +291,7 @@ uint16_t mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf **pkts,
 static int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq);
 
 /**
- * Query LKey from a packet buffer for Rx. No need to flush local caches
+ * Query LKey for an address on Rx. No need to flush local caches
  * as the Rx mempool database entries are valid for the lifetime of the queue.
  *
  * @param rxq
@@ -320,7 +320,40 @@ mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
                                     mp, addr);
 }
 
-#define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+/**
+ * Query LKey from a packet buffer for Rx. No need to flush local caches
+ * as the Rx mempool database entries are valid for the lifetime of the queue.
+ *
+ * @param rxq
+ *   Pointer to Rx queue structure.
+ * @param mb
+ *   Buffer to search the address of.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ *   This function always succeeds on valid input.
+ */
+static __rte_always_inline uint32_t
+mlx5_rx_mb2mr(struct mlx5_rxq_data *rxq, struct rte_mbuf *mb)
+{
+       struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+       uintptr_t addr = (uintptr_t)mb->buf_addr;
+       struct mlx5_rxq_ctrl *rxq_ctrl;
+       uint32_t lkey;
+
+       /* Linear search on MR cache array. */
+       lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
+                                  MLX5_MR_CACHE_N, addr);
+       if (likely(lkey != UINT32_MAX))
+               return lkey;
+       /*
+        * Slower search in the mempool database on miss.
+        * During queue creation rxq->sh is not yet set, so we use rxq_ctrl.
+        */
+       rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+       return mlx5_mr_mempool2mr_bh(&rxq_ctrl->sh->cdev->mr_scache,
+                                    mr_ctrl, mb->pool, addr);
+}
 
 /**
  * Convert timestamp from HW format to linear counter