1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2016 6WIND S.A.
3 * Copyright 2016 Mellanox.
7 #pragma GCC diagnostic ignored "-Wpedantic"
9 #include <infiniband/verbs.h>
11 #pragma GCC diagnostic error "-Wpedantic"
14 #include <rte_mempool.h>
15 #include <rte_malloc.h>
18 #include "mlx5_rxtx.h"
19 #include "mlx5_glue.h"
21 struct mlx5_check_mempool_data {
27 /* Called by mlx5_check_mempool() when iterating the memory chunks. */
29 mlx5_check_mempool_cb(struct rte_mempool *mp,
30 void *opaque, struct rte_mempool_memhdr *memhdr,
33 struct mlx5_check_mempool_data *data = opaque;
38 /* It already failed, skip the next chunks. */
41 /* It is the first chunk. */
42 if (data->start == NULL && data->end == NULL) {
43 data->start = memhdr->addr;
44 data->end = data->start + memhdr->len;
47 if (data->end == memhdr->addr) {
48 data->end += memhdr->len;
51 if (data->start == (char *)memhdr->addr + memhdr->len) {
52 data->start -= memhdr->len;
55 /* Error, mempool is not virtually contiguous. */
60 * Check if a mempool can be used: it must be virtually contiguous.
63 * Pointer to memory pool.
65 * Pointer to the start address of the mempool virtual memory area
67 * Pointer to the end address of the mempool virtual memory area
70 * 0 on success (mempool is virtually contiguous), -1 on error.
72 static int mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start,
75 struct mlx5_check_mempool_data data;
77 memset(&data, 0, sizeof(data));
78 rte_mempool_mem_iter(mp, mlx5_check_mempool_cb, &data);
79 *start = (uintptr_t)data.start;
80 *end = (uintptr_t)data.end;
86 * Register a Memory Region (MR) <-> Memory Pool (MP) association in
87 * txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
89 * This function should only be called by txq_mp2mr().
92 * Pointer to private structure.
94 * Pointer to TX queue structure.
96 * Memory Pool for which a Memory Region lkey must be returned.
98 * Index of the next available entry.
101 * mr on success, NULL on failure.
104 priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq,
105 struct rte_mempool *mp, unsigned int idx)
107 struct mlx5_txq_ctrl *txq_ctrl =
108 container_of(txq, struct mlx5_txq_ctrl, txq);
111 /* Add a new entry, register MR first. */
112 DEBUG("%p: discovered new memory pool \"%s\" (%p)",
113 (void *)txq_ctrl, mp->name, (void *)mp);
114 mr = priv_mr_get(priv, mp);
116 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
117 DEBUG("Using unregistered mempool 0x%p(%s) in secondary process,"
118 " please create mempool before rte_eth_dev_start()",
119 (void *)mp, mp->name);
122 mr = priv_mr_new(priv, mp);
124 if (unlikely(mr == NULL)) {
125 DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.",
129 if (unlikely(idx == RTE_DIM(txq->mp2mr))) {
130 /* Table is full, remove oldest entry. */
131 DEBUG("%p: MR <-> MP table full, dropping oldest entry.",
134 priv_mr_release(priv, txq->mp2mr[0]);
135 memmove(&txq->mp2mr[0], &txq->mp2mr[1],
136 (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0])));
138 /* Store the new entry. */
139 txq_ctrl->txq.mp2mr[idx] = mr;
140 DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32,
141 (void *)txq_ctrl, mp->name, (void *)mp,
142 txq_ctrl->txq.mp2mr[idx]->lkey);
147 * Register a Memory Region (MR) <-> Memory Pool (MP) association in
148 * txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
150 * This function should only be called by txq_mp2mr().
153 * Pointer to TX queue structure.
155 * Memory Pool for which a Memory Region lkey must be returned.
157 * Index of the next available entry.
160 * mr on success, NULL on failure.
163 mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp,
166 struct mlx5_txq_ctrl *txq_ctrl =
167 container_of(txq, struct mlx5_txq_ctrl, txq);
170 priv_lock(txq_ctrl->priv);
171 mr = priv_txq_mp2mr_reg(txq_ctrl->priv, txq, mp, idx);
172 priv_unlock(txq_ctrl->priv);
176 struct mlx5_mp2mr_mbuf_check_data {
181 * Callback function for rte_mempool_obj_iter() to check whether a given
182 * mempool object looks like a mbuf.
185 * The mempool pointer
187 * Context data (struct txq_mp2mr_mbuf_check_data). Contains the
192 * Object index, unused.
195 txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj,
196 uint32_t index __rte_unused)
198 struct mlx5_mp2mr_mbuf_check_data *data = arg;
199 struct rte_mbuf *buf = obj;
202 * Check whether mbuf structure fits element size and whether mempool
205 if (sizeof(*buf) > mp->elt_size || buf->pool != mp)
210 * Iterator function for rte_mempool_walk() to register existing mempools and
211 * fill the MP to MR cache of a TX queue.
214 * Memory Pool to register.
216 * Pointer to TX queue structure.
219 mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg)
221 struct priv *priv = (struct priv *)arg;
222 struct mlx5_mp2mr_mbuf_check_data data = {
227 /* Register mempool only if the first element looks like a mbuf. */
228 if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
231 mr = priv_mr_get(priv, mp);
233 priv_mr_release(priv, mr);
236 priv_mr_new(priv, mp);
240 * Register a new memory region from the mempool and store it in the memory
244 * Pointer to private structure.
246 * Pointer to the memory pool to register.
248 * The memory region on success.
251 priv_mr_new(struct priv *priv, struct rte_mempool *mp)
253 const struct rte_memseg *ms = rte_eal_get_physmem_layout();
259 mr = rte_zmalloc_socket(__func__, sizeof(*mr), 0, mp->socket_id);
261 DEBUG("unable to configure MR, ibv_reg_mr() failed.");
264 if (mlx5_check_mempool(mp, &start, &end) != 0) {
265 ERROR("mempool %p: not virtually contiguous",
269 DEBUG("mempool %p area start=%p end=%p size=%zu",
270 (void *)mp, (void *)start, (void *)end,
271 (size_t)(end - start));
272 /* Save original addresses for exact MR lookup. */
275 /* Round start and end to page boundary if found in memory segments. */
276 for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
277 uintptr_t addr = (uintptr_t)ms[i].addr;
278 size_t len = ms[i].len;
279 unsigned int align = ms[i].hugepage_sz;
281 if ((start > addr) && (start < addr + len))
282 start = RTE_ALIGN_FLOOR(start, align);
283 if ((end > addr) && (end < addr + len))
284 end = RTE_ALIGN_CEIL(end, align);
286 DEBUG("mempool %p using start=%p end=%p size=%zu for MR",
287 (void *)mp, (void *)start, (void *)end,
288 (size_t)(end - start));
289 mr->mr = mlx5_glue->reg_mr(priv->pd, (void *)start, end - start,
290 IBV_ACCESS_LOCAL_WRITE);
292 mr->lkey = rte_cpu_to_be_32(mr->mr->lkey);
293 rte_atomic32_inc(&mr->refcnt);
294 DEBUG("%p: new Memory Region %p refcnt: %d", (void *)priv,
295 (void *)mr, rte_atomic32_read(&mr->refcnt));
296 LIST_INSERT_HEAD(&priv->mr, mr, next);
301 * Search the memory region object in the memory region list.
304 * Pointer to private structure.
306 * Pointer to the memory pool to register.
308 * The memory region on success.
311 priv_mr_get(struct priv *priv, struct rte_mempool *mp)
316 if (LIST_EMPTY(&priv->mr))
318 LIST_FOREACH(mr, &priv->mr, next) {
320 rte_atomic32_inc(&mr->refcnt);
321 DEBUG("Memory Region %p refcnt: %d",
322 (void *)mr, rte_atomic32_read(&mr->refcnt));
330 * Release the memory region object.
333 * Pointer to memory region to release.
336 * 0 on success, errno on failure.
339 priv_mr_release(struct priv *priv, struct mlx5_mr *mr)
343 DEBUG("Memory Region %p refcnt: %d",
344 (void *)mr, rte_atomic32_read(&mr->refcnt));
345 if (rte_atomic32_dec_and_test(&mr->refcnt)) {
346 claim_zero(mlx5_glue->dereg_mr(mr->mr));
347 LIST_REMOVE(mr, next);
355 * Verify the flow list is empty
358 * Pointer to private structure.
360 * @return the number of object not released.
363 priv_mr_verify(struct priv *priv)
368 LIST_FOREACH(mr, &priv->mr, next) {
369 DEBUG("%p: mr %p still referenced", (void *)priv,