struct mlx5_mempool_mr *mrs;
/** Number of memory regions. */
unsigned int mrs_n;
+ /** Whether the MR were created for external pinned memory. */
+ bool is_extmem;
};
void
*
* @param mr_ctrl
* Pointer to MR control structure.
- * @param cdev
- * Pointer to the mlx5 device structure.
+ * @param dev_gen_ptr
+ * Pointer to generation number of global cache.
* @param socket
* NUMA socket on which memory must be allocated.
*
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
int
-mlx5_mr_ctrl_init(struct mlx5_mr_ctrl *mr_ctrl, struct mlx5_common_device *cdev,
+mlx5_mr_ctrl_init(struct mlx5_mr_ctrl *mr_ctrl, uint32_t *dev_gen_ptr,
int socket)
{
if (mr_ctrl == NULL) {
rte_errno = EINVAL;
return -rte_errno;
}
- mr_ctrl->cdev = cdev;
/* Save pointer of global generation number to check memory event. */
- mr_ctrl->dev_gen_ptr = &cdev->mr_scache.dev_gen;
+ mr_ctrl->dev_gen_ptr = dev_gen_ptr;
/* Initialize B-tree and allocate memory for bottom-half cache table. */
return mlx5_mr_btree_init(&mr_ctrl->cache_bh, MLX5_MR_BTREE_CACHE_N,
socket);
return;
DRV_LOG(DEBUG, "freeing MR(%p):", (void *)mr);
dereg_mr_cb(&mr->pmd_mr);
- if (mr->ms_bmp != NULL)
- rte_bitmap_free(mr->ms_bmp);
+ rte_bitmap_free(mr->ms_bmp);
mlx5_free(mr);
}
mr_lookup_caches(struct mlx5_mr_ctrl *mr_ctrl,
struct mr_cache_entry *entry, uintptr_t addr)
{
- struct mlx5_mr_share_cache *share_cache = &mr_ctrl->cdev->mr_scache;
+ struct mlx5_mr_share_cache *share_cache =
+ container_of(mr_ctrl->dev_gen_ptr, struct mlx5_mr_share_cache,
+ dev_gen);
+ struct mlx5_common_device *cdev =
+ container_of(share_cache, struct mlx5_common_device, mr_scache);
struct mlx5_mr_btree *bt = &mr_ctrl->cache_bh;
uint32_t lkey;
uint16_t idx;
}
rte_rwlock_read_unlock(&share_cache->rwlock);
/* First time to see the address? Create a new MR. */
- lkey = mlx5_mr_create(mr_ctrl->cdev, share_cache, entry, addr);
+ lkey = mlx5_mr_create(cdev, share_cache, entry, addr);
/*
* Update the local cache if successfully created a new global MR. Even
* if failed to create one, there's no action to take in this datapath
mlx5_mempool_get_chunks(struct rte_mempool *mp, struct mlx5_range **out,
unsigned int *out_n)
{
- struct mlx5_range *chunks;
unsigned int n;
+ DRV_LOG(DEBUG, "Collecting chunks of regular mempool %s", mp->name);
n = mp->nb_mem_chunks;
- chunks = calloc(sizeof(chunks[0]), n);
- if (chunks == NULL)
+ *out = calloc(sizeof(**out), n);
+ if (*out == NULL)
return -1;
- rte_mempool_mem_iter(mp, mlx5_range_from_mempool_chunk, chunks);
- *out = chunks;
+ rte_mempool_mem_iter(mp, mlx5_range_from_mempool_chunk, *out);
*out_n = n;
return 0;
}
{
struct mlx5_mempool_get_extmem_data data;
+ DRV_LOG(DEBUG, "Recovering external pinned pages of mempool %s",
+ mp->name);
memset(&data, 0, sizeof(data));
rte_mempool_obj_iter(mp, mlx5_mempool_get_extmem_cb, &data);
- if (data.ret < 0)
- return -1;
*out = data.heap;
*out_n = data.heap_size;
- return 0;
+ return data.ret;
}
/**
*
* @param[in] mp
* Analyzed mempool.
+ * @param[in] is_extmem
+ * Whether the pool is contains only external pinned buffers.
* @param[out] out
* Receives the ranges, caller must release it with free().
- * @param[out] ount_n
+ * @param[out] out_n
* Receives the number of @p out elements.
*
* @return
* 0 on success, (-1) on failure.
*/
static int
-mlx5_get_mempool_ranges(struct rte_mempool *mp, struct mlx5_range **out,
- unsigned int *out_n)
+mlx5_get_mempool_ranges(struct rte_mempool *mp, bool is_extmem,
+ struct mlx5_range **out, unsigned int *out_n)
{
struct mlx5_range *chunks;
unsigned int chunks_n, contig_n, i;
int ret;
/* Collect the pool underlying memory. */
- ret = (rte_pktmbuf_priv_flags(mp) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) ?
- mlx5_mempool_get_extmem(mp, &chunks, &chunks_n) :
- mlx5_mempool_get_chunks(mp, &chunks, &chunks_n);
+ ret = is_extmem ? mlx5_mempool_get_extmem(mp, &chunks, &chunks_n) :
+ mlx5_mempool_get_chunks(mp, &chunks, &chunks_n);
if (ret < 0)
return ret;
/* Merge adjacent chunks and place them at the beginning. */
*
* @param[in] mp
* Mempool to analyze.
+ * @param[in] is_extmem
+ * Whether the pool is contains only external pinned buffers.
* @param[out] out
* Receives memory ranges to register, aligned to the system page size.
* The caller must release them with free().
* 0 on success, (-1) on failure.
*/
static int
-mlx5_mempool_reg_analyze(struct rte_mempool *mp, struct mlx5_range **out,
- unsigned int *out_n, bool *share_hugepage)
+mlx5_mempool_reg_analyze(struct rte_mempool *mp, bool is_extmem,
+ struct mlx5_range **out, unsigned int *out_n,
+ bool *share_hugepage)
{
struct mlx5_range *ranges = NULL;
unsigned int i, ranges_n = 0;
struct rte_memseg_list *msl;
- if (mlx5_get_mempool_ranges(mp, &ranges, &ranges_n) < 0) {
+ if (mlx5_get_mempool_ranges(mp, is_extmem, &ranges, &ranges_n) < 0) {
DRV_LOG(ERR, "Cannot get address ranges for mempool %s",
mp->name);
return -1;
/** Create a registration object for the mempool. */
static struct mlx5_mempool_reg *
-mlx5_mempool_reg_create(struct rte_mempool *mp, unsigned int mrs_n)
+mlx5_mempool_reg_create(struct rte_mempool *mp, unsigned int mrs_n,
+ bool is_extmem)
{
struct mlx5_mempool_reg *mpr = NULL;
mpr = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO,
- sizeof(*mpr) + mrs_n * sizeof(mpr->mrs[0]),
+ sizeof(struct mlx5_mempool_reg),
RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
if (mpr == NULL) {
DRV_LOG(ERR, "Cannot allocate mempool %s registration object",
mp->name);
return NULL;
}
+ mpr->mrs = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO,
+ mrs_n * sizeof(struct mlx5_mempool_mr),
+ RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+ if (!mpr->mrs) {
+ DRV_LOG(ERR, "Cannot allocate mempool %s registration MRs",
+ mp->name);
+ mlx5_free(mpr);
+ return NULL;
+ }
mpr->mp = mp;
- mpr->mrs = (struct mlx5_mempool_mr *)(mpr + 1);
mpr->mrs_n = mrs_n;
+ mpr->is_extmem = is_extmem;
return mpr;
}
* Destroy a mempool registration object.
*
* @param standalone
- * Whether @p mpr owns its MRs excludively, i.e. they are not shared.
+ * Whether @p mpr owns its MRs exclusively, i.e. they are not shared.
*/
static void
mlx5_mempool_reg_destroy(struct mlx5_mr_share_cache *share_cache,
for (i = 0; i < mpr->mrs_n; i++)
share_cache->dereg_mr_cb(&mpr->mrs[i].pmd_mr);
+ mlx5_free(mpr->mrs);
}
mlx5_free(mpr);
}
static int
mlx5_mr_mempool_register_primary(struct mlx5_mr_share_cache *share_cache,
- void *pd, struct rte_mempool *mp)
+ void *pd, struct rte_mempool *mp,
+ bool is_extmem)
{
struct mlx5_range *ranges = NULL;
- struct mlx5_mempool_reg *mpr, *new_mpr;
+ struct mlx5_mempool_reg *mpr, *old_mpr, *new_mpr;
unsigned int i, ranges_n;
- bool share_hugepage;
+ bool share_hugepage, standalone = false;
int ret = -1;
/* Early check to avoid unnecessary creation of MRs. */
rte_rwlock_read_lock(&share_cache->rwlock);
- mpr = mlx5_mempool_reg_lookup(share_cache, mp);
+ old_mpr = mlx5_mempool_reg_lookup(share_cache, mp);
rte_rwlock_read_unlock(&share_cache->rwlock);
- if (mpr != NULL) {
+ if (old_mpr != NULL && (!is_extmem || old_mpr->is_extmem)) {
DRV_LOG(DEBUG, "Mempool %s is already registered for PD %p",
mp->name, pd);
rte_errno = EEXIST;
goto exit;
}
- if (mlx5_mempool_reg_analyze(mp, &ranges, &ranges_n,
+ if (mlx5_mempool_reg_analyze(mp, is_extmem, &ranges, &ranges_n,
&share_hugepage) < 0) {
DRV_LOG(ERR, "Cannot get mempool %s memory ranges", mp->name);
rte_errno = ENOMEM;
goto exit;
}
- new_mpr = mlx5_mempool_reg_create(mp, ranges_n);
+ new_mpr = mlx5_mempool_reg_create(mp, ranges_n, is_extmem);
if (new_mpr == NULL) {
DRV_LOG(ERR,
"Cannot create a registration object for mempool %s in PD %p",
/* Concurrent registration is not supposed to happen. */
rte_rwlock_write_lock(&share_cache->rwlock);
mpr = mlx5_mempool_reg_lookup(share_cache, mp);
+ if (mpr == old_mpr && old_mpr != NULL) {
+ LIST_REMOVE(old_mpr, next);
+ standalone = mlx5_mempool_reg_detach(mpr);
+ /* No need to flush the cache: old MRs cannot be in use. */
+ mpr = NULL;
+ }
if (mpr == NULL) {
mlx5_mempool_reg_attach(new_mpr);
LIST_INSERT_HEAD(&share_cache->mempool_reg_list, new_mpr, next);
mlx5_mempool_reg_destroy(share_cache, new_mpr, true);
rte_errno = EEXIST;
goto exit;
+ } else if (old_mpr != NULL) {
+ DRV_LOG(DEBUG, "Mempool %s registration for PD %p updated for external memory",
+ mp->name, pd);
+ mlx5_mempool_reg_destroy(share_cache, old_mpr, standalone);
}
exit:
free(ranges);
static int
mlx5_mr_mempool_register_secondary(struct mlx5_common_device *cdev,
- struct rte_mempool *mp)
+ struct rte_mempool *mp, bool is_extmem)
{
- return mlx5_mp_req_mempool_reg(cdev, mp, true);
+ return mlx5_mp_req_mempool_reg(cdev, mp, true, is_extmem);
}
/**
*/
int
mlx5_mr_mempool_register(struct mlx5_common_device *cdev,
- struct rte_mempool *mp)
+ struct rte_mempool *mp, bool is_extmem)
{
if (mp->flags & RTE_MEMPOOL_F_NON_IO)
return 0;
switch (rte_eal_process_type()) {
case RTE_PROC_PRIMARY:
return mlx5_mr_mempool_register_primary(&cdev->mr_scache,
- cdev->pd, mp);
+ cdev->pd, mp,
+ is_extmem);
case RTE_PROC_SECONDARY:
- return mlx5_mr_mempool_register_secondary(cdev, mp);
+ return mlx5_mr_mempool_register_secondary(cdev, mp, is_extmem);
default:
return -1;
}
mlx5_mr_mempool_unregister_secondary(struct mlx5_common_device *cdev,
struct rte_mempool *mp)
{
- return mlx5_mp_req_mempool_reg(cdev, mp, false);
+ return mlx5_mp_req_mempool_reg(cdev, mp, false, false /* is_extmem */);
}
/**
for (i = 0; i < mpr->mrs_n; i++) {
const struct mlx5_pmd_mr *mr = &mpr->mrs[i].pmd_mr;
- uintptr_t mr_addr = (uintptr_t)mr->addr;
+ uintptr_t mr_start = (uintptr_t)mr->addr;
+ uintptr_t mr_end = mr_start + mr->len;
- if (mr_addr <= addr) {
+ if (mr_start <= addr && addr < mr_end) {
lkey = rte_cpu_to_be_32(mr->lkey);
- entry->start = mr_addr;
- entry->end = mr_addr + mr->len;
+ entry->start = mr_start;
+ entry->end = mr_end;
entry->lkey = lkey;
break;
}
/**
* Update bottom-half cache from the list of mempool registrations.
*
- * @param share_cache
- * Pointer to a global shared MR cache.
* @param mr_ctrl
* Per-queue MR control handle.
* @param entry
* MR lkey on success, UINT32_MAX on failure.
*/
static uint32_t
-mlx5_lookup_mempool_regs(struct mlx5_mr_share_cache *share_cache,
- struct mlx5_mr_ctrl *mr_ctrl,
+mlx5_lookup_mempool_regs(struct mlx5_mr_ctrl *mr_ctrl,
struct mr_cache_entry *entry,
struct rte_mempool *mp, uintptr_t addr)
{
+ struct mlx5_mr_share_cache *share_cache =
+ container_of(mr_ctrl->dev_gen_ptr, struct mlx5_mr_share_cache,
+ dev_gen);
struct mlx5_mr_btree *bt = &mr_ctrl->cache_bh;
struct mlx5_mempool_reg *mpr;
uint32_t lkey = UINT32_MAX;
return lkey;
}
+/**
+ * Populate cache with LKeys of all MRs used by the mempool.
+ * It is intended to be used to register Rx mempools in advance.
+ *
+ * @param mr_ctrl
+ * Per-queue MR control handle.
+ * @param mp
+ * Registered memory pool.
+ *
+ * @return
+ * 0 on success, (-1) on failure and rte_errno is set.
+ */
+int
+mlx5_mr_mempool_populate_cache(struct mlx5_mr_ctrl *mr_ctrl,
+ struct rte_mempool *mp)
+{
+ struct mlx5_mr_share_cache *share_cache =
+ container_of(mr_ctrl->dev_gen_ptr, struct mlx5_mr_share_cache,
+ dev_gen);
+ struct mlx5_mr_btree *bt = &mr_ctrl->cache_bh;
+ struct mlx5_mempool_reg *mpr;
+ unsigned int i;
+
+ /*
+ * Registration is valid after the lock is released,
+ * because the function is called after the mempool is registered.
+ */
+ rte_rwlock_read_lock(&share_cache->rwlock);
+ mpr = mlx5_mempool_reg_lookup(share_cache, mp);
+ rte_rwlock_read_unlock(&share_cache->rwlock);
+ if (mpr == NULL) {
+ DRV_LOG(ERR, "Mempool %s is not registered", mp->name);
+ rte_errno = ENOENT;
+ return -1;
+ }
+ for (i = 0; i < mpr->mrs_n; i++) {
+ struct mlx5_mempool_mr *mr = &mpr->mrs[i];
+ struct mr_cache_entry entry;
+ uint32_t lkey;
+ uint16_t idx;
+
+ lkey = mr_btree_lookup(bt, &idx, (uintptr_t)mr->pmd_mr.addr);
+ if (lkey != UINT32_MAX)
+ continue;
+ if (bt->len == bt->size)
+ mr_btree_expand(bt, bt->size << 1);
+ entry.start = (uintptr_t)mr->pmd_mr.addr;
+ entry.end = entry.start + mr->pmd_mr.len;
+ entry.lkey = rte_cpu_to_be_32(mr->pmd_mr.lkey);
+ if (mr_btree_insert(bt, &entry) < 0) {
+ DRV_LOG(ERR, "Cannot insert cache entry for mempool %s MR %08x",
+ mp->name, entry.lkey);
+ rte_errno = EINVAL;
+ return -1;
+ }
+ }
+ return 0;
+}
+
/**
* Bottom-half lookup for the address from the mempool.
*
- * @param share_cache
- * Pointer to a global shared MR cache.
* @param mr_ctrl
* Per-queue MR control handle.
* @param mp
* MR lkey on success, UINT32_MAX on failure.
*/
uint32_t
-mlx5_mr_mempool2mr_bh(struct mlx5_mr_share_cache *share_cache,
- struct mlx5_mr_ctrl *mr_ctrl,
+mlx5_mr_mempool2mr_bh(struct mlx5_mr_ctrl *mr_ctrl,
struct rte_mempool *mp, uintptr_t addr)
{
struct mr_cache_entry *repl = &mr_ctrl->cache[mr_ctrl->head];
if (likely(lkey != UINT32_MAX)) {
*repl = (*mr_ctrl->cache_bh.table)[bh_idx];
} else {
- lkey = mlx5_lookup_mempool_regs(share_cache, mr_ctrl, repl,
- mp, addr);
+ lkey = mlx5_lookup_mempool_regs(mr_ctrl, repl, mp, addr);
/* Can only fail if the address is not from the mempool. */
if (unlikely(lkey == UINT32_MAX))
return UINT32_MAX;
uint32_t
mlx5_mr_mb2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mb)
{
+ struct rte_mempool *mp;
+ struct mlx5_mprq_buf *buf;
uint32_t lkey;
uintptr_t addr = (uintptr_t)mb->buf_addr;
- struct mlx5_common_device *cdev = mr_ctrl->cdev;
-
- if (cdev->config.mr_mempool_reg_en) {
- struct rte_mempool *mp = NULL;
- struct mlx5_mprq_buf *buf;
-
- if (!RTE_MBUF_HAS_EXTBUF(mb)) {
- mp = mlx5_mb2mp(mb);
- } else if (mb->shinfo->free_cb == mlx5_mprq_buf_free_cb) {
- /* Recover MPRQ mempool. */
- buf = mb->shinfo->fcb_opaque;
- mp = buf->mp;
- }
- if (mp != NULL) {
- lkey = mlx5_mr_mempool2mr_bh(&cdev->mr_scache,
- mr_ctrl, mp, addr);
- /*
- * Lookup can only fail on invalid input, e.g. "addr"
- * is not from "mp" or "mp" has MEMPOOL_F_NON_IO set.
- */
- if (lkey != UINT32_MAX)
- return lkey;
- }
- /* Fallback for generic mechanism in corner cases. */
+ struct mlx5_mr_share_cache *share_cache =
+ container_of(mr_ctrl->dev_gen_ptr, struct mlx5_mr_share_cache,
+ dev_gen);
+ struct mlx5_common_device *cdev =
+ container_of(share_cache, struct mlx5_common_device, mr_scache);
+ bool external, mprq, pinned = false;
+
+ /* Recover MPRQ mempool. */
+ external = RTE_MBUF_HAS_EXTBUF(mb);
+ if (external && mb->shinfo->free_cb == mlx5_mprq_buf_free_cb) {
+ mprq = true;
+ buf = mb->shinfo->fcb_opaque;
+ mp = buf->mp;
+ } else {
+ mprq = false;
+ mp = mlx5_mb2mp(mb);
+ pinned = rte_pktmbuf_priv_flags(mp) &
+ RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF;
+ }
+ if (!external || mprq || pinned) {
+ lkey = mlx5_mr_mempool2mr_bh(mr_ctrl, mp, addr);
+ if (lkey != UINT32_MAX)
+ return lkey;
+ /* MPRQ is always registered. */
+ MLX5_ASSERT(!mprq);
+ }
+ /* Register pinned external memory if the mempool is not used for Rx. */
+ if (cdev->config.mr_mempool_reg_en && pinned) {
+ if (mlx5_mr_mempool_register(cdev, mp, true) < 0)
+ return UINT32_MAX;
+ lkey = mlx5_mr_mempool2mr_bh(mr_ctrl, mp, addr);
+ MLX5_ASSERT(lkey != UINT32_MAX);
+ return lkey;
}
+ /* Fallback to generic mechanism in corner cases. */
return mlx5_mr_addr2mr_bh(mr_ctrl, addr);
}
git.droids-corp.org / dpdk.git / blobdiff