1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
8 * Memory management functions for mlx4 driver.
18 /* Verbs headers do not support -pedantic. */
20 #pragma GCC diagnostic ignored "-Wpedantic"
22 #include <infiniband/verbs.h>
24 #pragma GCC diagnostic error "-Wpedantic"
27 #include <rte_branch_prediction.h>
28 #include <rte_common.h>
29 #include <rte_errno.h>
30 #include <rte_malloc.h>
31 #include <rte_memory.h>
32 #include <rte_mempool.h>
33 #include <rte_spinlock.h>
35 #include "mlx4_glue.h"
36 #include "mlx4_rxtx.h"
37 #include "mlx4_utils.h"
39 struct mlx4_check_mempool_data {
46 * Called by mlx4_check_mempool() when iterating the memory chunks.
49 * Pointer to memory pool (unused).
50 * @param[in, out] data
51 * Pointer to shared buffer with mlx4_check_mempool().
53 * Pointer to mempool chunk header.
55 * Mempool element index (unused).
58 mlx4_check_mempool_cb(struct rte_mempool *mp, void *opaque,
59 struct rte_mempool_memhdr *memhdr,
62 struct mlx4_check_mempool_data *data = opaque;
66 /* It already failed, skip the next chunks. */
69 /* It is the first chunk. */
70 if (data->start == NULL && data->end == NULL) {
71 data->start = memhdr->addr;
72 data->end = data->start + memhdr->len;
75 if (data->end == memhdr->addr) {
76 data->end += memhdr->len;
79 if (data->start == (char *)memhdr->addr + memhdr->len) {
80 data->start -= memhdr->len;
83 /* Error, mempool is not virtually contiguous. */
88 * Check if a mempool can be used: it must be virtually contiguous.
91 * Pointer to memory pool.
93 * Pointer to the start address of the mempool virtual memory area.
95 * Pointer to the end address of the mempool virtual memory area.
98 * 0 on success (mempool is virtually contiguous), -1 on error.
101 mlx4_check_mempool(struct rte_mempool *mp, uintptr_t *start, uintptr_t *end)
103 struct mlx4_check_mempool_data data;
105 memset(&data, 0, sizeof(data));
106 rte_mempool_mem_iter(mp, mlx4_check_mempool_cb, &data);
107 *start = (uintptr_t)data.start;
108 *end = (uintptr_t)data.end;
113 * Obtain a memory region from a memory pool.
115 * If a matching memory region already exists, it is returned with its
116 * reference count incremented, otherwise a new one is registered.
119 * Pointer to private structure.
121 * Pointer to memory pool.
124 * Memory region pointer, NULL in case of error and rte_errno is set.
127 mlx4_mr_get(struct priv *priv, struct rte_mempool *mp)
129 const struct rte_memseg *ms = rte_eal_get_physmem_layout();
135 if (mlx4_check_mempool(mp, &start, &end) != 0) {
137 ERROR("mempool %p: not virtually contiguous",
141 DEBUG("mempool %p area start=%p end=%p size=%zu",
142 (void *)mp, (void *)start, (void *)end,
143 (size_t)(end - start));
144 /* Round start and end to page boundary if found in memory segments. */
145 for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
146 uintptr_t addr = (uintptr_t)ms[i].addr;
147 size_t len = ms[i].len;
148 unsigned int align = ms[i].hugepage_sz;
150 if ((start > addr) && (start < addr + len))
151 start = RTE_ALIGN_FLOOR(start, align);
152 if ((end > addr) && (end < addr + len))
153 end = RTE_ALIGN_CEIL(end, align);
155 DEBUG("mempool %p using start=%p end=%p size=%zu for MR",
156 (void *)mp, (void *)start, (void *)end,
157 (size_t)(end - start));
158 rte_spinlock_lock(&priv->mr_lock);
159 LIST_FOREACH(mr, &priv->mr, next)
160 if (mp == mr->mp && start >= mr->start && end <= mr->end)
166 mr = rte_malloc(__func__, sizeof(*mr), 0);
171 *mr = (struct mlx4_mr){
176 .mr = mlx4_glue->reg_mr(priv->pd, (void *)start, end - start,
177 IBV_ACCESS_LOCAL_WRITE),
181 mr->lkey = mr->mr->lkey;
182 LIST_INSERT_HEAD(&priv->mr, mr, next);
186 rte_errno = errno ? errno : EINVAL;
189 rte_spinlock_unlock(&priv->mr_lock);
194 * Release a memory region.
196 * This function decrements its reference count and destroys it after
199 * Note to avoid race conditions given this function may be used from the
200 * data plane, it's extremely important that each user holds its own
204 * Memory region to release.
207 mlx4_mr_put(struct mlx4_mr *mr)
209 struct priv *priv = mr->priv;
211 rte_spinlock_lock(&priv->mr_lock);
215 LIST_REMOVE(mr, next);
216 claim_zero(mlx4_glue->dereg_mr(mr->mr));
219 rte_spinlock_unlock(&priv->mr_lock);
223 * Add memory region (MR) <-> memory pool (MP) association to txq->mp2mr[].
224 * If mp2mr[] is full, remove an entry first.
227 * Pointer to Tx queue structure.
229 * Memory pool for which a memory region lkey must be added.
231 * Index in memory pool (MP) where to add memory region (MR).
234 * Added mr->lkey on success, (uint32_t)-1 on failure.
237 mlx4_txq_add_mr(struct txq *txq, struct rte_mempool *mp, uint32_t i)
241 /* Add a new entry, register MR first. */
242 DEBUG("%p: discovered new memory pool \"%s\" (%p)",
243 (void *)txq, mp->name, (void *)mp);
244 mr = mlx4_mr_get(txq->priv, mp);
245 if (unlikely(mr == NULL)) {
246 DEBUG("%p: unable to configure MR, mlx4_mr_get() failed",
250 if (unlikely(i == RTE_DIM(txq->mp2mr))) {
251 /* Table is full, remove oldest entry. */
252 DEBUG("%p: MR <-> MP table full, dropping oldest entry.",
255 mlx4_mr_put(txq->mp2mr[0].mr);
256 memmove(&txq->mp2mr[0], &txq->mp2mr[1],
257 (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0])));
259 /* Store the new entry. */
260 txq->mp2mr[i].mp = mp;
261 txq->mp2mr[i].mr = mr;
262 txq->mp2mr[i].lkey = mr->lkey;
263 DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32,
264 (void *)txq, mp->name, (void *)mp, txq->mp2mr[i].lkey);
265 return txq->mp2mr[i].lkey;