rte_errno = EINVAL;
return -rte_errno;
}
+ assert(!bt->table && !bt->size);
memset(bt, 0, sizeof(*bt));
bt->table = rte_calloc_socket("B-tree table",
n, sizeof(struct mlx5_mr_cache),
0, socket);
if (bt->table == NULL) {
rte_errno = ENOMEM;
- DRV_LOG(ERR,
- "failed to allocate memory for btree cache on socket %d",
- socket);
+ DEBUG("failed to allocate memory for btree cache on socket %d",
+ socket);
return -rte_errno;
}
bt->size = n;
(*bt->table)[bt->len++] = (struct mlx5_mr_cache) {
.lkey = UINT32_MAX,
};
- DRV_LOG(DEBUG, "initialized B-tree %p with table %p",
- (void *)bt, (void *)bt->table);
+ DEBUG("initialized B-tree %p with table %p",
+ (void *)bt, (void *)bt->table);
return 0;
}
{
if (bt == NULL)
return;
- DRV_LOG(DEBUG, "freeing B-tree %p with table %p",
- (void *)bt, (void *)bt->table);
+ DEBUG("freeing B-tree %p with table %p",
+ (void *)bt, (void *)bt->table);
rte_free(bt->table);
memset(bt, 0, sizeof(*bt));
}
* @param bt
* Pointer to B-tree structure.
*/
-static void
-mlx5_mr_btree_dump(struct mlx5_mr_btree *bt)
+void
+mlx5_mr_btree_dump(struct mlx5_mr_btree *bt __rte_unused)
{
+#ifndef NDEBUG
int idx;
struct mlx5_mr_cache *lkp_tbl;
for (idx = 0; idx < bt->len; ++idx) {
struct mlx5_mr_cache *entry = &lkp_tbl[idx];
- DRV_LOG(DEBUG,
- "B-tree(%p)[%u],"
- " [0x%" PRIxPTR ", 0x%" PRIxPTR ") lkey=0x%x",
- (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ DEBUG("B-tree(%p)[%u],"
+ " [0x%" PRIxPTR ", 0x%" PRIxPTR ") lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
}
+#endif
}
/**
uintptr_t end = 0;
uint32_t idx = 0;
+ /* MR for external memory doesn't have memseg list. */
+ if (mr->msl == NULL) {
+ struct ibv_mr *ibv_mr = mr->ibv_mr;
+
+ assert(mr->ms_bmp_n == 1);
+ assert(mr->ms_n == 1);
+ assert(base_idx == 0);
+ /*
+ * Can't search it from memseg list but get it directly from
+ * verbs MR as there's only one chunk.
+ */
+ entry->start = (uintptr_t)ibv_mr->addr;
+ entry->end = (uintptr_t)ibv_mr->addr + mr->ibv_mr->length;
+ entry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);
+ /* Returning 1 ends iteration. */
+ return 1;
+ }
for (idx = base_idx; idx < mr->ms_bmp_n; ++idx) {
if (rte_bitmap_get(mr->ms_bmp, idx)) {
const struct rte_memseg_list *msl;
DRV_LOG(DEBUG, "port %u inserting MR(%p) to global cache",
dev->data->port_id, (void *)mr);
for (n = 0; n < mr->ms_bmp_n; ) {
- struct mlx5_mr_cache entry = { 0, };
+ struct mlx5_mr_cache entry;
+ memset(&entry, 0, sizeof(entry));
/* Find a contiguous chunk and advance the index. */
n = mr_find_next_chunk(mr, &entry, n);
if (!entry.end)
if (mr->ms_n == 0)
continue;
for (n = 0; n < mr->ms_bmp_n; ) {
- struct mlx5_mr_cache ret = { 0, };
+ struct mlx5_mr_cache ret;
+ memset(&ret, 0, sizeof(ret));
n = mr_find_next_chunk(mr, &ret, n);
if (addr >= ret.start && addr < ret.end) {
/* Found. */
* Find out a contiguous virtual address chunk in use, to which the
* given address belongs, in order to register maximum range. In the
* best case where mempools are not dynamically recreated and
- * '--socket-mem' is speicified as an EAL option, it is very likely to
+ * '--socket-mem' is specified as an EAL option, it is very likely to
* have only one MR(LKey) per a socket and per a hugepage-size even
* though the system memory is highly fragmented.
*/
assert(msl->page_sz == ms->hugepage_sz);
/* Number of memsegs in the range. */
ms_n = len / msl->page_sz;
- DRV_LOG(DEBUG,
- "port %u extending %p to [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
- " page_sz=0x%" PRIx64 ", ms_n=%u",
- dev->data->port_id, (void *)addr,
- data.start, data.end, msl->page_sz, ms_n);
+ DEBUG("port %u extending %p to [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
+ " page_sz=0x%" PRIx64 ", ms_n=%u",
+ dev->data->port_id, (void *)addr,
+ data.start, data.end, msl->page_sz, ms_n);
/* Size of memory for bitmap. */
bmp_size = rte_bitmap_get_memory_footprint(ms_n);
mr = rte_zmalloc_socket(NULL,
bmp_size,
RTE_CACHE_LINE_SIZE, msl->socket_id);
if (mr == NULL) {
- DRV_LOG(WARNING,
- "port %u unable to allocate memory for a new MR of"
- " address (%p).",
- dev->data->port_id, (void *)addr);
+ DEBUG("port %u unable to allocate memory for a new MR of"
+ " address (%p).",
+ dev->data->port_id, (void *)addr);
rte_errno = ENOMEM;
goto err_nolock;
}
bmp_mem = RTE_PTR_ALIGN_CEIL(mr + 1, RTE_CACHE_LINE_SIZE);
mr->ms_bmp = rte_bitmap_init(ms_n, bmp_mem, bmp_size);
if (mr->ms_bmp == NULL) {
- DRV_LOG(WARNING,
- "port %u unable to initialize bitamp for a new MR of"
- " address (%p).",
- dev->data->port_id, (void *)addr);
+ DEBUG("port %u unable to initialize bitamp for a new MR of"
+ " address (%p).",
+ dev->data->port_id, (void *)addr);
rte_errno = EINVAL;
goto err_nolock;
}
data_re = data;
if (len > msl->page_sz &&
!rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data_re)) {
- DRV_LOG(WARNING,
- "port %u unable to find virtually contiguous"
- " chunk for address (%p)."
- " rte_memseg_contig_walk() failed.",
- dev->data->port_id, (void *)addr);
+ DEBUG("port %u unable to find virtually contiguous"
+ " chunk for address (%p)."
+ " rte_memseg_contig_walk() failed.",
+ dev->data->port_id, (void *)addr);
rte_errno = ENXIO;
goto err_memlock;
}
* here again.
*/
mr_btree_insert(&priv->mr.cache, entry);
- DRV_LOG(DEBUG,
- "port %u found MR for %p on final lookup, abort",
- dev->data->port_id, (void *)addr);
+ DEBUG("port %u found MR for %p on final lookup, abort",
+ dev->data->port_id, (void *)addr);
rte_rwlock_write_unlock(&priv->mr.rwlock);
rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
/*
*/
for (n = 0; n < ms_n; ++n) {
uintptr_t start;
- struct mlx5_mr_cache ret = { 0, };
+ struct mlx5_mr_cache ret;
+ memset(&ret, 0, sizeof(ret));
start = data_re.start + n * msl->page_sz;
/* Exclude memsegs already registered by other MRs. */
if (mr_lookup_dev(dev, &ret, start) == UINT32_MAX) {
mr->ibv_mr = mlx5_glue->reg_mr(priv->pd, (void *)data.start, len,
IBV_ACCESS_LOCAL_WRITE);
if (mr->ibv_mr == NULL) {
- DRV_LOG(WARNING,
- "port %u fail to create a verbs MR for address (%p)",
- dev->data->port_id, (void *)addr);
+ DEBUG("port %u fail to create a verbs MR for address (%p)",
+ dev->data->port_id, (void *)addr);
rte_errno = EINVAL;
goto err_mrlock;
}
assert((uintptr_t)mr->ibv_mr->addr == data.start);
assert(mr->ibv_mr->length == len);
LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
- DRV_LOG(DEBUG,
- "port %u MR CREATED (%p) for %p:\n"
- " [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
- " lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
- dev->data->port_id, (void *)mr, (void *)addr,
- data.start, data.end, rte_cpu_to_be_32(mr->ibv_mr->lkey),
- mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
+ DEBUG("port %u MR CREATED (%p) for %p:\n"
+ " [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
+ " lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
+ dev->data->port_id, (void *)mr, (void *)addr,
+ data.start, data.end, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
/* Insert to the global cache table. */
mr_insert_dev_cache(dev, mr);
/* Fill in output data. */
int i;
int rebuild = 0;
- DRV_LOG(DEBUG, "port %u free callback: addr=%p, len=%zu",
- dev->data->port_id, addr, len);
+ DEBUG("port %u free callback: addr=%p, len=%zu",
+ dev->data->port_id, addr, len);
msl = rte_mem_virt2memseg_list(addr);
/* addr and len must be page-aligned. */
assert((uintptr_t)addr == RTE_ALIGN((uintptr_t)addr, msl->page_sz));
mr = mr_lookup_dev_list(dev, &entry, start);
if (mr == NULL)
continue;
+ assert(mr->msl); /* Can't be external memory. */
ms = rte_mem_virt2memseg((void *)start, msl);
assert(ms != NULL);
assert(msl->page_sz == ms->hugepage_sz);
pos = ms_idx - mr->ms_base_idx;
assert(rte_bitmap_get(mr->ms_bmp, pos));
assert(pos < mr->ms_bmp_n);
- DRV_LOG(DEBUG, "port %u MR(%p): clear bitmap[%u] for addr %p",
- dev->data->port_id, (void *)mr, pos, (void *)start);
+ DEBUG("port %u MR(%p): clear bitmap[%u] for addr %p",
+ dev->data->port_id, (void *)mr, pos, (void *)start);
rte_bitmap_clear(mr->ms_bmp, pos);
if (--mr->ms_n == 0) {
LIST_REMOVE(mr, mr);
LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
- DRV_LOG(DEBUG, "port %u remove MR(%p) from list",
- dev->data->port_id, (void *)mr);
+ DEBUG("port %u remove MR(%p) from list",
+ dev->data->port_id, (void *)mr);
}
/*
* MR is fragmented or will be freed. the global cache must be
* before the core sees the newly allocated memory.
*/
++priv->mr.dev_gen;
- DRV_LOG(DEBUG, "broadcasting local cache flush, gen=%d",
- priv->mr.dev_gen);
+ DEBUG("broadcasting local cache flush, gen=%d",
+ priv->mr.dev_gen);
rte_smp_wmb();
}
rte_rwlock_write_unlock(&priv->mr.rwlock);
- if (rebuild && rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
- mlx5_mr_dump_dev(dev);
}
/**
* @return
* Searched LKey on success, UINT32_MAX on no match.
*/
-uint32_t
+static uint32_t
mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr)
{
struct mlx5_txq_ctrl *txq_ctrl =
return mlx5_mr_addr2mr_bh(ETH_DEV(priv), mr_ctrl, addr);
}
+/**
+ * Bottom-half of LKey search on Tx. If it can't be searched in the memseg
+ * list, register the mempool of the mbuf as externally allocated memory.
+ *
+ * @param txq
+ * Pointer to Tx queue structure.
+ * @param mb
+ * Pointer to mbuf.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+uint32_t
+mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
+{
+ uintptr_t addr = (uintptr_t)mb->buf_addr;
+ uint32_t lkey;
+
+ lkey = mlx5_tx_addr2mr_bh(txq, addr);
+ if (lkey == UINT32_MAX && rte_errno == ENXIO) {
+ /* Mempool may have externally allocated memory. */
+ return mlx5_tx_update_ext_mp(txq, addr, mlx5_mb2mp(mb));
+ }
+ return lkey;
+}
+
/**
* Flush all of the local cache entries.
*
(void *)mr_ctrl, mr_ctrl->cur_gen);
}
+/**
+ * Called during rte_mempool_mem_iter() by mlx5_mr_update_ext_mp().
+ *
+ * Externally allocated chunk is registered and a MR is created for the chunk.
+ * The MR object is added to the global list. If memseg list of a MR object
+ * (mr->msl) is null, the MR object can be regarded as externally allocated
+ * memory.
+ *
+ * Once external memory is registered, it should be static. If the memory is
+ * freed and the virtual address range has different physical memory mapped
+ * again, it may cause crash on device due to the wrong translation entry. PMD
+ * can't track the free event of the external memory for now.
+ */
+static void
+mlx5_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque,
+ struct rte_mempool_memhdr *memhdr,
+ unsigned mem_idx __rte_unused)
+{
+ struct mr_update_mp_data *data = opaque;
+ struct rte_eth_dev *dev = data->dev;
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr_ctrl *mr_ctrl = data->mr_ctrl;
+ struct mlx5_mr *mr = NULL;
+ uintptr_t addr = (uintptr_t)memhdr->addr;
+ size_t len = memhdr->len;
+ struct mlx5_mr_cache entry;
+ uint32_t lkey;
+
+ /* If already registered, it should return. */
+ rte_rwlock_read_lock(&priv->mr.rwlock);
+ lkey = mr_lookup_dev(dev, &entry, addr);
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+ if (lkey != UINT32_MAX)
+ return;
+ mr = rte_zmalloc_socket(NULL,
+ RTE_ALIGN_CEIL(sizeof(*mr),
+ RTE_CACHE_LINE_SIZE),
+ RTE_CACHE_LINE_SIZE, mp->socket_id);
+ if (mr == NULL) {
+ DRV_LOG(WARNING,
+ "port %u unable to allocate memory for a new MR of"
+ " mempool (%s).",
+ dev->data->port_id, mp->name);
+ data->ret = -1;
+ return;
+ }
+ DRV_LOG(DEBUG, "port %u register MR for chunk #%d of mempool (%s)",
+ dev->data->port_id, mem_idx, mp->name);
+ mr->ibv_mr = mlx5_glue->reg_mr(priv->pd, (void *)addr, len,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (mr->ibv_mr == NULL) {
+ DRV_LOG(WARNING,
+ "port %u fail to create a verbs MR for address (%p)",
+ dev->data->port_id, (void *)addr);
+ rte_free(mr);
+ data->ret = -1;
+ return;
+ }
+ mr->msl = NULL; /* Mark it is external memory. */
+ mr->ms_bmp = NULL;
+ mr->ms_n = 1;
+ mr->ms_bmp_n = 1;
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
+ DRV_LOG(DEBUG,
+ "port %u MR CREATED (%p) for external memory %p:\n"
+ " [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
+ " lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
+ dev->data->port_id, (void *)mr, (void *)addr,
+ addr, addr + len, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
+ /* Insert to the global cache table. */
+ mr_insert_dev_cache(dev, mr);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ /* Insert to the local cache table */
+ mlx5_mr_addr2mr_bh(dev, mr_ctrl, addr);
+}
+
+/**
+ * Register MR for entire memory chunks in a Mempool having externally allocated
+ * memory and fill in local cache.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param mp
+ * Pointer to registering Mempool.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static uint32_t
+mlx5_mr_update_ext_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+ struct rte_mempool *mp)
+{
+ struct mr_update_mp_data data = {
+ .dev = dev,
+ .mr_ctrl = mr_ctrl,
+ .ret = 0,
+ };
+
+ rte_mempool_mem_iter(mp, mlx5_mr_update_ext_mp_cb, &data);
+ return data.ret;
+}
+
+/**
+ * Register MR entire memory chunks in a Mempool having externally allocated
+ * memory and search LKey of the address to return.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param addr
+ * Search key.
+ * @param mp
+ * Pointer to registering Mempool where addr belongs.
+ *
+ * @return
+ * LKey for address on success, UINT32_MAX on failure.
+ */
+uint32_t
+mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr,
+ struct rte_mempool *mp)
+{
+ struct mlx5_txq_ctrl *txq_ctrl =
+ container_of(txq, struct mlx5_txq_ctrl, txq);
+ struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+ struct priv *priv = txq_ctrl->priv;
+
+ mlx5_mr_update_ext_mp(ETH_DEV(priv), mr_ctrl, mp);
+ return mlx5_tx_addr2mr_bh(txq, addr);
+}
+
/* Called during rte_mempool_mem_iter() by mlx5_mr_update_mp(). */
static void
mlx5_mr_update_mp_cb(struct rte_mempool *mp __rte_unused, void *opaque,
};
rte_mempool_mem_iter(mp, mlx5_mr_update_mp_cb, &data);
+ if (data.ret < 0 && rte_errno == ENXIO) {
+ /* Mempool may have externally allocated memory. */
+ return mlx5_mr_update_ext_mp(dev, mr_ctrl, mp);
+ }
return data.ret;
}
* Pointer to Ethernet device.
*/
void
-mlx5_mr_dump_dev(struct rte_eth_dev *dev)
+mlx5_mr_dump_dev(struct rte_eth_dev *dev __rte_unused)
{
+#ifndef NDEBUG
struct priv *priv = dev->data->dev_private;
struct mlx5_mr *mr;
int mr_n = 0;
LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
unsigned int n;
- DRV_LOG(DEBUG,
- "port %u MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
- dev->data->port_id, mr_n++,
- rte_cpu_to_be_32(mr->ibv_mr->lkey),
- mr->ms_n, mr->ms_bmp_n);
+ DEBUG("port %u MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
+ dev->data->port_id, mr_n++,
+ rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ mr->ms_n, mr->ms_bmp_n);
if (mr->ms_n == 0)
continue;
for (n = 0; n < mr->ms_bmp_n; ) {
n = mr_find_next_chunk(mr, &ret, n);
if (!ret.end)
break;
- DRV_LOG(DEBUG,
- " chunk[%u], [0x%" PRIxPTR ", 0x%" PRIxPTR ")",
- chunk_n++, ret.start, ret.end);
+ DEBUG(" chunk[%u], [0x%" PRIxPTR ", 0x%" PRIxPTR ")",
+ chunk_n++, ret.start, ret.end);
}
}
- DRV_LOG(DEBUG, "port %u dumping global cache", dev->data->port_id);
+ DEBUG("port %u dumping global cache", dev->data->port_id);
mlx5_mr_btree_dump(&priv->mr.cache);
rte_rwlock_read_unlock(&priv->mr.rwlock);
+#endif
}
/**