1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2016 6WIND S.A.
3 * Copyright 2016 Mellanox.
7 #pragma GCC diagnostic ignored "-Wpedantic"
9 #include <infiniband/verbs.h>
11 #pragma GCC diagnostic error "-Wpedantic"
14 #include <rte_mempool.h>
15 #include <rte_malloc.h>
18 #include "mlx5_rxtx.h"
19 #include "mlx5_glue.h"
21 struct mlx5_check_mempool_data {
27 /* Called by mlx5_check_mempool() when iterating the memory chunks. */
29 mlx5_check_mempool_cb(struct rte_mempool *mp __rte_unused,
30 void *opaque, struct rte_mempool_memhdr *memhdr,
31 unsigned int mem_idx __rte_unused)
33 struct mlx5_check_mempool_data *data = opaque;
35 /* It already failed, skip the next chunks. */
38 /* It is the first chunk. */
39 if (data->start == NULL && data->end == NULL) {
40 data->start = memhdr->addr;
41 data->end = data->start + memhdr->len;
44 if (data->end == memhdr->addr) {
45 data->end += memhdr->len;
48 if (data->start == (char *)memhdr->addr + memhdr->len) {
49 data->start -= memhdr->len;
52 /* Error, mempool is not virtually contiguous. */
57 * Check if a mempool can be used: it must be virtually contiguous.
60 * Pointer to memory pool.
62 * Pointer to the start address of the mempool virtual memory area
64 * Pointer to the end address of the mempool virtual memory area
67 * 0 on success (mempool is virtually contiguous), -1 on error.
70 mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start,
73 struct mlx5_check_mempool_data data;
75 memset(&data, 0, sizeof(data));
76 rte_mempool_mem_iter(mp, mlx5_check_mempool_cb, &data);
77 *start = (uintptr_t)data.start;
78 *end = (uintptr_t)data.end;
83 * Register a Memory Region (MR) <-> Memory Pool (MP) association in
84 * txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
87 * Pointer to TX queue structure.
89 * Memory Pool for which a Memory Region lkey must be returned.
91 * Index of the next available entry.
94 * mr on success, NULL on failure and rte_errno is set.
97 mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp,
100 struct mlx5_txq_ctrl *txq_ctrl =
101 container_of(txq, struct mlx5_txq_ctrl, txq);
102 struct rte_eth_dev *dev;
105 rte_spinlock_lock(&txq_ctrl->priv->mr_lock);
106 /* Add a new entry, register MR first. */
107 DEBUG("port %u discovered new memory pool \"%s\" (%p)",
108 txq_ctrl->priv->dev->data->port_id, mp->name, (void *)mp);
109 dev = txq_ctrl->priv->dev;
110 mr = mlx5_mr_get(dev, mp);
112 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
113 DEBUG("port %u using unregistered mempool 0x%p(%s) in "
114 "secondary process, please create mempool before "
115 " rte_eth_dev_start()",
116 txq_ctrl->priv->dev->data->port_id,
117 (void *)mp, mp->name);
118 rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
122 mr = mlx5_mr_new(dev, mp);
124 if (unlikely(mr == NULL)) {
125 rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
126 DEBUG("port %u unable to configure memory region, ibv_reg_mr()"
128 txq_ctrl->priv->dev->data->port_id);
131 if (unlikely(idx == RTE_DIM(txq->mp2mr))) {
132 /* Table is full, remove oldest entry. */
133 DEBUG("port %u memroy region <-> memory pool table full, "
134 " dropping oldest entry",
135 txq_ctrl->priv->dev->data->port_id);
137 mlx5_mr_release(txq->mp2mr[0]);
138 memmove(&txq->mp2mr[0], &txq->mp2mr[1],
139 (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0])));
141 /* Store the new entry. */
142 txq_ctrl->txq.mp2mr[idx] = mr;
143 DEBUG("port %u new memory region lkey for MP \"%s\" (%p): 0x%08" PRIu32,
144 txq_ctrl->priv->dev->data->port_id, mp->name, (void *)mp,
145 txq_ctrl->txq.mp2mr[idx]->lkey);
146 rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
150 struct mlx5_mp2mr_mbuf_check_data {
155 * Callback function for rte_mempool_obj_iter() to check whether a given
156 * mempool object looks like a mbuf.
159 * The mempool pointer
161 * Context data (struct txq_mp2mr_mbuf_check_data). Contains the
166 * Object index, unused.
169 txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj,
170 uint32_t index __rte_unused)
172 struct mlx5_mp2mr_mbuf_check_data *data = arg;
173 struct rte_mbuf *buf = obj;
176 * Check whether mbuf structure fits element size and whether mempool
179 if (sizeof(*buf) > mp->elt_size || buf->pool != mp)
184 * Iterator function for rte_mempool_walk() to register existing mempools and
185 * fill the MP to MR cache of a TX queue.
188 * Memory Pool to register.
190 * Pointer to TX queue structure.
193 mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg)
195 struct priv *priv = (struct priv *)arg;
196 struct mlx5_mp2mr_mbuf_check_data data = {
201 /* Register mempool only if the first element looks like a mbuf. */
202 if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
205 mr = mlx5_mr_get(priv->dev, mp);
210 mr = mlx5_mr_new(priv->dev, mp);
212 ERROR("port %u cannot create memory region: %s",
213 priv->dev->data->port_id, strerror(rte_errno));
217 * Register a new memory region from the mempool and store it in the memory
221 * Pointer to Ethernet device.
223 * Pointer to the memory pool to register.
226 * The memory region on success, NULL on failure and rte_errno is set.
229 mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp)
231 struct priv *priv = dev->data->dev_private;
232 const struct rte_memseg *ms = rte_eal_get_physmem_layout();
238 mr = rte_zmalloc_socket(__func__, sizeof(*mr), 0, mp->socket_id);
240 DEBUG("port %u unable to configure memory region, ibv_reg_mr()"
246 if (mlx5_check_mempool(mp, &start, &end) != 0) {
247 ERROR("port %u mempool %p: not virtually contiguous",
248 dev->data->port_id, (void *)mp);
252 DEBUG("port %u mempool %p area start=%p end=%p size=%zu",
253 dev->data->port_id, (void *)mp, (void *)start, (void *)end,
254 (size_t)(end - start));
255 /* Save original addresses for exact MR lookup. */
258 /* Round start and end to page boundary if found in memory segments. */
259 for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
260 uintptr_t addr = (uintptr_t)ms[i].addr;
261 size_t len = ms[i].len;
262 unsigned int align = ms[i].hugepage_sz;
264 if ((start > addr) && (start < addr + len))
265 start = RTE_ALIGN_FLOOR(start, align);
266 if ((end > addr) && (end < addr + len))
267 end = RTE_ALIGN_CEIL(end, align);
269 DEBUG("port %u mempool %p using start=%p end=%p size=%zu for memory"
271 dev->data->port_id, (void *)mp, (void *)start, (void *)end,
272 (size_t)(end - start));
273 mr->mr = mlx5_glue->reg_mr(priv->pd, (void *)start, end - start,
274 IBV_ACCESS_LOCAL_WRITE);
280 mr->lkey = rte_cpu_to_be_32(mr->mr->lkey);
281 rte_atomic32_inc(&mr->refcnt);
282 DEBUG("port %u new memory region %p refcnt: %d",
283 dev->data->port_id, (void *)mr, rte_atomic32_read(&mr->refcnt));
284 LIST_INSERT_HEAD(&priv->mr, mr, next);
289 * Search the memory region object in the memory region list.
292 * Pointer to Ethernet device.
294 * Pointer to the memory pool to register.
297 * The memory region on success.
300 mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp)
302 struct priv *priv = dev->data->dev_private;
306 if (LIST_EMPTY(&priv->mr))
308 LIST_FOREACH(mr, &priv->mr, next) {
310 rte_atomic32_inc(&mr->refcnt);
311 DEBUG("port %u memory region %p refcnt: %d",
312 dev->data->port_id, (void *)mr,
313 rte_atomic32_read(&mr->refcnt));
321 * Release the memory region object.
324 * Pointer to memory region to release.
327 * 1 while a reference on it exists, 0 when freed.
330 mlx5_mr_release(struct mlx5_mr *mr)
333 DEBUG("memory region %p refcnt: %d", (void *)mr,
334 rte_atomic32_read(&mr->refcnt));
335 if (rte_atomic32_dec_and_test(&mr->refcnt)) {
336 claim_zero(mlx5_glue->dereg_mr(mr->mr));
337 LIST_REMOVE(mr, next);
345 * Verify the flow list is empty
348 * Pointer to Ethernet device.
351 * The number of object not released.
354 mlx5_mr_verify(struct rte_eth_dev *dev)
356 struct priv *priv = dev->data->dev_private;
360 LIST_FOREACH(mr, &priv->mr, next) {
361 DEBUG("port %u memory region %p still referenced",
362 dev->data->port_id, (void *)mr);