- "port %u unable to find virtually contiguous"
- " chunk for address (%p)."
- " rte_memseg_contig_walk() failed.",
- dev->data->port_id, (void *)addr);
- rte_errno = ENXIO;
- goto err_nolock;
- }
-alloc_resources:
- /* Addresses must be page-aligned. */
- assert(rte_is_aligned((void *)data.start, data.msl->page_sz));
- assert(rte_is_aligned((void *)data.end, data.msl->page_sz));
- msl = data.msl;
- ms = rte_mem_virt2memseg((void *)data.start, msl);
- len = data.end - data.start;
- assert(msl->page_sz == ms->hugepage_sz);
- /* Number of memsegs in the range. */
- ms_n = len / msl->page_sz;
- DEBUG("port %u extending %p to [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
- " page_sz=0x%" PRIx64 ", ms_n=%u",
- dev->data->port_id, (void *)addr,
- data.start, data.end, msl->page_sz, ms_n);
- /* Size of memory for bitmap. */
- bmp_size = rte_bitmap_get_memory_footprint(ms_n);
- mr = rte_zmalloc_socket(NULL,
- RTE_ALIGN_CEIL(sizeof(*mr),
- RTE_CACHE_LINE_SIZE) +
- bmp_size,
- RTE_CACHE_LINE_SIZE, msl->socket_id);
- if (mr == NULL) {
- DEBUG("port %u unable to allocate memory for a new MR of"
- " address (%p).",
- dev->data->port_id, (void *)addr);
- rte_errno = ENOMEM;
- goto err_nolock;
- }
- mr->msl = msl;
- /*
- * Save the index of the first memseg and initialize memseg bitmap. To
- * see if a memseg of ms_idx in the memseg-list is still valid, check:
- * rte_bitmap_get(mr->bmp, ms_idx - mr->ms_base_idx)
- */
- mr->ms_base_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
- bmp_mem = RTE_PTR_ALIGN_CEIL(mr + 1, RTE_CACHE_LINE_SIZE);
- mr->ms_bmp = rte_bitmap_init(ms_n, bmp_mem, bmp_size);
- if (mr->ms_bmp == NULL) {
- DEBUG("port %u unable to initialize bitamp for a new MR of"
- " address (%p).",
- dev->data->port_id, (void *)addr);
- rte_errno = EINVAL;
- goto err_nolock;
- }
- /*
- * Should recheck whether the extended contiguous chunk is still valid.
- * Because memory_hotplug_lock can't be held if there's any memory
- * related calls in a critical path, resource allocation above can't be
- * locked. If the memory has been changed at this point, try again with
- * just single page. If not, go on with the big chunk atomically from
- * here.
- */
- rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
- data_re = data;
- if (len > msl->page_sz &&
- !rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data_re)) {
- DEBUG("port %u unable to find virtually contiguous"
- " chunk for address (%p)."
- " rte_memseg_contig_walk() failed.",
- dev->data->port_id, (void *)addr);
- rte_errno = ENXIO;
- goto err_memlock;
- }
- if (data.start != data_re.start || data.end != data_re.end) {
- /*
- * The extended contiguous chunk has been changed. Try again
- * with single memseg instead.
- */
- data.start = RTE_ALIGN_FLOOR(addr, msl->page_sz);
- data.end = data.start + msl->page_sz;
- rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
- mr_free(mr);
- goto alloc_resources;
- }
- assert(data.msl == data_re.msl);
- rte_rwlock_write_lock(&priv->mr.rwlock);
- /*
- * Check the address is really missing. If other thread already created
- * one or it is not found due to overflow, abort and return.
- */
- if (mr_lookup_dev(dev, entry, addr) != UINT32_MAX) {
- /*
- * Insert to the global cache table. It may fail due to
- * low-on-memory. Then, this entry will have to be searched
- * here again.
- */
- mr_btree_insert(&priv->mr.cache, entry);
- DEBUG("port %u found MR for %p on final lookup, abort",
- dev->data->port_id, (void *)addr);
- rte_rwlock_write_unlock(&priv->mr.rwlock);
- rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
- /*
- * Must be unlocked before calling rte_free() because
- * mlx5_mr_mem_event_free_cb() can be called inside.
- */
- mr_free(mr);
- return entry->lkey;
- }
- /*
- * Trim start and end addresses for verbs MR. Set bits for registering
- * memsegs but exclude already registered ones. Bitmap can be
- * fragmented.
- */
- for (n = 0; n < ms_n; ++n) {
- uintptr_t start;
- struct mlx5_mr_cache ret = { 0, };
-
- start = data_re.start + n * msl->page_sz;
- /* Exclude memsegs already registered by other MRs. */
- if (mr_lookup_dev(dev, &ret, start) == UINT32_MAX) {
- /*
- * Start from the first unregistered memseg in the
- * extended range.
- */
- if (ms_idx_shift == -1) {
- mr->ms_base_idx += n;
- data.start = start;
- ms_idx_shift = n;
- }
- data.end = start + msl->page_sz;
- rte_bitmap_set(mr->ms_bmp, n - ms_idx_shift);
- ++mr->ms_n;
- }
- }
- len = data.end - data.start;
- mr->ms_bmp_n = len / msl->page_sz;
- assert(ms_idx_shift + mr->ms_bmp_n <= ms_n);
- /*
- * Finally create a verbs MR for the memory chunk. ibv_reg_mr() can be
- * called with holding the memory lock because it doesn't use
- * mlx5_alloc_buf_extern() which eventually calls rte_malloc_socket()
- * through mlx5_alloc_verbs_buf().
- */
- mr->ibv_mr = mlx5_glue->reg_mr(priv->pd, (void *)data.start, len,
- IBV_ACCESS_LOCAL_WRITE);
- if (mr->ibv_mr == NULL) {
- DEBUG("port %u fail to create a verbs MR for address (%p)",
- dev->data->port_id, (void *)addr);
- rte_errno = EINVAL;
- goto err_mrlock;