sz->trailer_size = new_size - sz->header_size - sz->elt_size;
}
- if (! rte_eal_has_hugepages()) {
- /*
- * compute trailer size so that pool elements fit exactly in
- * a standard page
- */
- int page_size = getpagesize();
- int new_size = page_size - sz->header_size - sz->elt_size;
- if (new_size < 0 || (unsigned int)new_size < sz->trailer_size) {
- printf("When hugepages are disabled, pool objects "
- "can't exceed PAGE_SIZE: %d + %d + %d > %d\n",
- sz->header_size, sz->elt_size, sz->trailer_size,
- page_size);
- return 0;
- }
- sz->trailer_size = new_size;
- }
-
/* this is the size of an object, including header and trailer */
sz->total_size = sz->header_size + sz->elt_size + sz->trailer_size;
return (size_t)paddr_idx << pg_shift;
}
-#ifndef RTE_LIBRTE_XEN_DOM0
-/* stub if DOM0 support not configured */
-struct rte_mempool *
-rte_dom0_mempool_create(const char *name __rte_unused,
- unsigned n __rte_unused,
- unsigned elt_size __rte_unused,
- unsigned cache_size __rte_unused,
- unsigned private_data_size __rte_unused,
- rte_mempool_ctor_t *mp_init __rte_unused,
- void *mp_init_arg __rte_unused,
- rte_mempool_obj_ctor_t *obj_init __rte_unused,
- void *obj_init_arg __rte_unused,
- int socket_id __rte_unused,
- unsigned flags __rte_unused)
-{
- rte_errno = EINVAL;
- return NULL;
-}
-#endif
-
-/* create the mempool */
-struct rte_mempool *
-rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
- unsigned cache_size, unsigned private_data_size,
- rte_mempool_ctor_t *mp_init, void *mp_init_arg,
- rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg,
- int socket_id, unsigned flags)
-{
- if (rte_xen_dom0_supported())
- return rte_dom0_mempool_create(name, n, elt_size,
- cache_size, private_data_size,
- mp_init, mp_init_arg,
- obj_init, obj_init_arg,
- socket_id, flags);
- else
- return rte_mempool_xmem_create(name, n, elt_size,
- cache_size, private_data_size,
- mp_init, mp_init_arg,
- obj_init, obj_init_arg,
- socket_id, flags,
- NULL, NULL, MEMPOOL_PG_NUM_DEFAULT,
- MEMPOOL_PG_SHIFT_MAX);
-}
-
/* create the internal ring */
static int
rte_mempool_ring_create(struct rte_mempool *mp)
return cnt;
}
-/* Default function to populate the mempool: allocate memory in mezones,
+/* Populate the mempool with a virtual area. Return the number of
+ * objects added, or a negative value on error.
+ */
+static int
+rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
+ size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
+ void *opaque)
+{
+ phys_addr_t paddr;
+ size_t off, phys_len;
+ int ret, cnt = 0;
+
+ /* mempool must not be populated */
+ if (mp->nb_mem_chunks != 0)
+ return -EEXIST;
+ /* address and len must be page-aligned */
+ if (RTE_PTR_ALIGN_CEIL(addr, pg_sz) != addr)
+ return -EINVAL;
+ if (RTE_ALIGN_CEIL(len, pg_sz) != len)
+ return -EINVAL;
+
+ for (off = 0; off + pg_sz <= len &&
+ mp->populated_size < mp->size; off += phys_len) {
+
+ paddr = rte_mem_virt2phy(addr + off);
+ /* required for xen_dom0 to get the machine address */
+ paddr = rte_mem_phy2mch(-1, paddr);
+
+ if (paddr == RTE_BAD_PHYS_ADDR) {
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ /* populate with the largest group of contiguous pages */
+ for (phys_len = pg_sz; off + phys_len < len; phys_len += pg_sz) {
+ phys_addr_t paddr_tmp;
+
+ paddr_tmp = rte_mem_virt2phy(addr + off + phys_len);
+ paddr_tmp = rte_mem_phy2mch(-1, paddr_tmp);
+
+ if (paddr_tmp != paddr + phys_len)
+ break;
+ }
+
+ ret = rte_mempool_populate_phys(mp, addr + off, paddr,
+ phys_len, free_cb, opaque);
+ if (ret < 0)
+ goto fail;
+ /* no need to call the free callback for next chunks */
+ free_cb = NULL;
+ cnt += ret;
+ }
+
+ return cnt;
+
+ fail:
+ rte_mempool_free_memchunks(mp);
+ return ret;
+}
+
+/* Default function to populate the mempool: allocate memory in memzones,
* and populate them. Return the number of objects added, or a negative
* value on error.
*/
-static int rte_mempool_populate_default(struct rte_mempool *mp)
+static int
+rte_mempool_populate_default(struct rte_mempool *mp)
{
int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY;
char mz_name[RTE_MEMZONE_NAMESIZE];
const struct rte_memzone *mz;
- size_t size, total_elt_sz, align;
+ size_t size, total_elt_sz, align, pg_sz, pg_shift;
unsigned mz_id, n;
int ret;
if (mp->nb_mem_chunks != 0)
return -EEXIST;
- align = RTE_CACHE_LINE_SIZE;
+ if (rte_eal_has_hugepages()) {
+ pg_shift = 0; /* not needed, zone is physically contiguous */
+ pg_sz = 0;
+ align = RTE_CACHE_LINE_SIZE;
+ } else {
+ pg_sz = getpagesize();
+ pg_shift = rte_bsf32(pg_sz);
+ align = pg_sz;
+ }
+
total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) {
- size = rte_mempool_xmem_size(n, total_elt_sz, 0);
+ size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift);
ret = snprintf(mz_name, sizeof(mz_name),
RTE_MEMPOOL_MZ_FORMAT "_%d", mp->name, mz_id);
goto fail;
}
- ret = rte_mempool_populate_phys(mp, mz->addr, mz->phys_addr,
- mz->len, rte_mempool_memchunk_mz_free,
- (void *)(uintptr_t)mz);
+ /* use memzone physical address if it is valid */
+ if (rte_eal_has_hugepages() && !rte_xen_dom0_supported())
+ ret = rte_mempool_populate_phys(mp, mz->addr,
+ mz->phys_addr, mz->len,
+ rte_mempool_memchunk_mz_free,
+ (void *)(uintptr_t)mz);
+ else
+ ret = rte_mempool_populate_virt(mp, mz->addr,
+ mz->len, pg_sz,
+ rte_mempool_memchunk_mz_free,
+ (void *)(uintptr_t)mz);
if (ret < 0)
goto fail;
}
return ret;
}
-/*
- * Create the mempool over already allocated chunk of memory.
- * That external memory buffer can consists of physically disjoint pages.
- * Setting vaddr to NULL, makes mempool to fallback to original behaviour
- * and allocate space for mempool and it's elements as one big chunk of
- * physically continuos memory.
- * */
-struct rte_mempool *
-rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size,
- unsigned cache_size, unsigned private_data_size,
- rte_mempool_ctor_t *mp_init, void *mp_init_arg,
- rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
- int socket_id, unsigned flags, void *vaddr,
- const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift)
+/* free a mempool */
+static void
+rte_mempool_free(struct rte_mempool *mp)
+{
+ struct rte_mempool_list *mempool_list = NULL;
+ struct rte_tailq_entry *te;
+
+ if (mp == NULL)
+ return;
+
+ mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list);
+ rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
+ /* find out tailq entry */
+ TAILQ_FOREACH(te, mempool_list, next) {
+ if (te->data == (void *)mp)
+ break;
+ }
+
+ if (te != NULL) {
+ TAILQ_REMOVE(mempool_list, te, next);
+ rte_free(te);
+ }
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ rte_mempool_free_memchunks(mp);
+ rte_ring_free(mp->ring);
+ rte_memzone_free(mp->mz);
+}
+
+/* create an empty mempool */
+static struct rte_mempool *
+rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
+ unsigned cache_size, unsigned private_data_size,
+ int socket_id, unsigned flags)
{
char mz_name[RTE_MEMZONE_NAMESIZE];
struct rte_mempool_list *mempool_list;
size_t mempool_size;
int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY;
struct rte_mempool_objsz objsz;
- int ret;
/* compilation-time checks */
RTE_BUILD_BUG_ON((sizeof(struct rte_mempool) &
return NULL;
}
- /* check that we have both VA and PA */
- if (vaddr != NULL && paddr == NULL) {
- rte_errno = EINVAL;
- return NULL;
- }
-
- /* Check that pg_num and pg_shift parameters are valid. */
- if (pg_num == 0 || pg_shift > MEMPOOL_PG_SHIFT_MAX) {
- rte_errno = EINVAL;
- return NULL;
- }
-
/* "no cache align" imply "no spread" */
if (flags & MEMPOOL_F_NO_CACHE_ALIGN)
flags |= MEMPOOL_F_NO_SPREAD;
goto exit_unlock;
}
- /*
- * If user provided an external memory buffer, then use it to
- * store mempool objects. Otherwise reserve a memzone that is large
- * enough to hold mempool header and metadata plus mempool objects.
- */
mempool_size = MEMPOOL_HEADER_SIZE(mp, cache_size);
mempool_size += private_data_size;
mempool_size = RTE_ALIGN_CEIL(mempool_size, RTE_MEMPOOL_ALIGN);
goto exit_unlock;
/* init the mempool structure */
+ mp = mz->addr;
memset(mp, 0, sizeof(*mp));
snprintf(mp->name, sizeof(mp->name), "%s", name);
- mp->phys_addr = mz->phys_addr;
+ mp->mz = mz;
mp->socket_id = socket_id;
mp->size = n;
mp->flags = flags;
+ mp->socket_id = socket_id;
mp->elt_size = objsz.elt_size;
mp->header_size = objsz.header_size;
mp->trailer_size = objsz.trailer_size;
mp->local_cache = (struct rte_mempool_cache *)
RTE_PTR_ADD(mp, MEMPOOL_HEADER_SIZE(mp, 0));
- /* call the initializer */
+ te->data = mp;
+ rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
+ TAILQ_INSERT_TAIL(mempool_list, te, next);
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+ rte_rwlock_write_unlock(RTE_EAL_MEMPOOL_RWLOCK);
+
+ return mp;
+
+exit_unlock:
+ rte_rwlock_write_unlock(RTE_EAL_MEMPOOL_RWLOCK);
+ rte_free(te);
+ rte_mempool_free(mp);
+ return NULL;
+}
+
+/* create the mempool */
+struct rte_mempool *
+rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
+ unsigned cache_size, unsigned private_data_size,
+ rte_mempool_ctor_t *mp_init, void *mp_init_arg,
+ rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
+ int socket_id, unsigned flags)
+{
+ struct rte_mempool *mp;
+
+ mp = rte_mempool_create_empty(name, n, elt_size, cache_size,
+ private_data_size, socket_id, flags);
+ if (mp == NULL)
+ return NULL;
+
+ /* call the mempool priv initializer */
if (mp_init)
mp_init(mp, mp_init_arg);
- /* mempool elements allocated together with mempool */
- if (vaddr == NULL)
- ret = rte_mempool_populate_default(mp);
- else
- ret = rte_mempool_populate_phys_tab(mp, vaddr,
- paddr, pg_num, pg_shift, NULL, NULL);
- if (ret < 0) {
- rte_errno = -ret;
- goto exit_unlock;
- } else if (ret != (int)mp->size) {
- rte_errno = EINVAL;
- goto exit_unlock;
- }
+ if (rte_mempool_populate_default(mp) < 0)
+ goto fail;
- /* call the initializer */
+ /* call the object initializers */
if (obj_init)
rte_mempool_obj_iter(mp, obj_init, obj_init_arg);
- te->data = (void *) mp;
+ return mp;
- rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
- TAILQ_INSERT_TAIL(mempool_list, te, next);
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
- rte_rwlock_write_unlock(RTE_EAL_MEMPOOL_RWLOCK);
+ fail:
+ rte_mempool_free(mp);
+ return NULL;
+}
- return mp;
+/*
+ * Create the mempool over already allocated chunk of memory.
+ * That external memory buffer can consists of physically disjoint pages.
+ * Setting vaddr to NULL, makes mempool to fallback to original behaviour
+ * and allocate space for mempool and it's elements as one big chunk of
+ * physically continuos memory.
+ */
+struct rte_mempool *
+rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size,
+ unsigned cache_size, unsigned private_data_size,
+ rte_mempool_ctor_t *mp_init, void *mp_init_arg,
+ rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
+ int socket_id, unsigned flags, void *vaddr,
+ const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift)
+{
+ struct rte_mempool *mp = NULL;
+ int ret;
-exit_unlock:
- rte_rwlock_write_unlock(RTE_EAL_MEMPOOL_RWLOCK);
- if (mp != NULL) {
- rte_mempool_free_memchunks(mp);
- rte_ring_free(mp->ring);
+ /* no virtual address supplied, use rte_mempool_create() */
+ if (vaddr == NULL)
+ return rte_mempool_create(name, n, elt_size, cache_size,
+ private_data_size, mp_init, mp_init_arg,
+ obj_init, obj_init_arg, socket_id, flags);
+
+ /* check that we have both VA and PA */
+ if (paddr == NULL) {
+ rte_errno = EINVAL;
+ return NULL;
}
- rte_free(te);
- if (mz != NULL)
- rte_memzone_free(mz);
+ /* Check that pg_shift parameter is valid. */
+ if (pg_shift > MEMPOOL_PG_SHIFT_MAX) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+
+ mp = rte_mempool_create_empty(name, n, elt_size, cache_size,
+ private_data_size, socket_id, flags);
+ if (mp == NULL)
+ return NULL;
+
+ /* call the mempool priv initializer */
+ if (mp_init)
+ mp_init(mp, mp_init_arg);
+
+ ret = rte_mempool_populate_phys_tab(mp, vaddr, paddr, pg_num, pg_shift,
+ NULL, NULL);
+ if (ret < 0 || ret != (int)mp->size)
+ goto fail;
+
+ /* call the object initializers */
+ if (obj_init)
+ rte_mempool_obj_iter(mp, obj_init, obj_init_arg);
+
+ return mp;
+
+ fail:
+ rte_mempool_free(mp);
return NULL;
}
fprintf(f, "mempool <%s>@%p\n", mp->name, mp);
fprintf(f, " flags=%x\n", mp->flags);
fprintf(f, " ring=<%s>@%p\n", mp->ring->name, mp->ring);
- fprintf(f, " phys_addr=0x%" PRIx64 "\n", mp->phys_addr);
+ fprintf(f, " phys_addr=0x%" PRIx64 "\n", mp->mz->phys_addr);
fprintf(f, " nb_mem_chunks=%u\n", mp->nb_mem_chunks);
fprintf(f, " size=%"PRIu32"\n", mp->size);
fprintf(f, " populated_size=%"PRIu32"\n", mp->populated_size);