* BSD LICENSE
*
* Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2016 6WIND S.A.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
return new_obj_size * RTE_MEMPOOL_ALIGN;
}
+/**
+ * A mempool object iterator callback function.
+ */
+typedef void (*rte_mempool_obj_iter_t)(void * /*obj_iter_arg*/,
+ void * /*obj_start*/,
+ void * /*obj_end*/,
+ uint32_t /*obj_index */,
+ phys_addr_t /*physaddr*/);
+
static void
-mempool_add_elem(struct rte_mempool *mp, void *obj, uint32_t obj_idx,
- rte_mempool_obj_cb_t *obj_init, void *obj_init_arg)
+mempool_add_elem(struct rte_mempool *mp, void *obj, phys_addr_t physaddr)
{
struct rte_mempool_objhdr *hdr;
struct rte_mempool_objtlr *tlr __rte_unused;
- obj = (char *)obj + mp->header_size;
-
/* set mempool ptr in header */
hdr = RTE_PTR_SUB(obj, sizeof(*hdr));
hdr->mp = mp;
+ hdr->physaddr = physaddr;
STAILQ_INSERT_TAIL(&mp->elt_list, hdr, next);
+ mp->populated_size++;
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE2;
tlr = __mempool_get_trailer(obj);
tlr->cookie = RTE_MEMPOOL_TRAILER_COOKIE;
#endif
- /* call the initializer */
- if (obj_init)
- obj_init(mp, obj_init_arg, obj, obj_idx);
/* enqueue in ring */
rte_ring_sp_enqueue(mp->ring, obj);
* chunk, invoke a callback. It returns the effective number of objects
* in this memory.
*/
-uint32_t
-rte_mempool_obj_iter(void *vaddr, uint32_t elt_num, size_t total_elt_sz,
+static uint32_t
+rte_mempool_obj_mem_iter(void *vaddr, uint32_t elt_num, size_t total_elt_sz,
size_t align, const phys_addr_t paddr[], uint32_t pg_num,
uint32_t pg_shift, rte_mempool_obj_iter_t obj_iter, void *obj_iter_arg)
{
uint32_t pgn, pgf;
uintptr_t end, start, va;
uintptr_t pg_sz;
+ phys_addr_t physaddr;
pg_sz = (uintptr_t)1 << pg_shift;
va = (uintptr_t)vaddr;
* otherwise, just skip that chunk unused.
*/
if (k == pgn) {
+ physaddr = paddr[k] + (start & (pg_sz - 1));
if (obj_iter != NULL)
obj_iter(obj_iter_arg, (void *)start,
- (void *)end, i);
+ (void *)end, i, physaddr);
va = end;
j += pgf;
i++;
return i;
}
-/*
- * Populate mempool with the objects.
- */
-
-struct mempool_populate_arg {
- struct rte_mempool *mp;
- rte_mempool_obj_cb_t *obj_init;
- void *obj_init_arg;
-};
-
-static void
-mempool_obj_populate(void *arg, void *start, void *end, uint32_t idx)
+/* call obj_cb() for each mempool element */
+uint32_t
+rte_mempool_obj_iter(struct rte_mempool *mp,
+ rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg)
{
- struct mempool_populate_arg *pa = arg;
+ struct rte_mempool_objhdr *hdr;
+ void *obj;
+ unsigned n = 0;
- mempool_add_elem(pa->mp, start, idx, pa->obj_init, pa->obj_init_arg);
- pa->mp->elt_va_end = (uintptr_t)end;
-}
+ STAILQ_FOREACH(hdr, &mp->elt_list, next) {
+ obj = (char *)hdr + sizeof(*hdr);
+ obj_cb(mp, obj_cb_arg, obj, n);
+ n++;
+ }
-static void
-mempool_populate(struct rte_mempool *mp, size_t num, size_t align,
- rte_mempool_obj_cb_t *obj_init, void *obj_init_arg)
-{
- uint32_t elt_sz;
- struct mempool_populate_arg arg;
-
- elt_sz = mp->elt_size + mp->header_size + mp->trailer_size;
- arg.mp = mp;
- arg.obj_init = obj_init;
- arg.obj_init_arg = obj_init_arg;
-
- mp->size = rte_mempool_obj_iter((void *)mp->elt_va_start,
- num, elt_sz, align,
- mp->elt_pa, mp->pg_num, mp->pg_shift,
- mempool_obj_populate, &arg);
+ return n;
}
/* get the header, trailer and total size of a mempool element. */
*/
static void
mempool_lelem_iter(void *arg, __rte_unused void *start, void *end,
- __rte_unused uint32_t idx)
+ __rte_unused uint32_t idx, __rte_unused phys_addr_t physaddr)
{
*(uintptr_t *)arg = (uintptr_t)end;
}
va = (uintptr_t)vaddr;
uv = va;
- if ((n = rte_mempool_obj_iter(vaddr, elt_num, total_elt_sz, 1,
+ if ((n = rte_mempool_obj_mem_iter(vaddr, elt_num, total_elt_sz, 1,
paddr, pg_num, pg_shift, mempool_lelem_iter,
&uv)) != elt_num) {
return -(ssize_t)n;
MEMPOOL_PG_SHIFT_MAX);
}
+/* create the internal ring */
+static int
+rte_mempool_ring_create(struct rte_mempool *mp)
+{
+ int rg_flags = 0;
+ char rg_name[RTE_RING_NAMESIZE];
+ struct rte_ring *r;
+
+ snprintf(rg_name, sizeof(rg_name), RTE_MEMPOOL_MZ_FORMAT, mp->name);
+
+ /* ring flags */
+ if (mp->flags & MEMPOOL_F_SP_PUT)
+ rg_flags |= RING_F_SP_ENQ;
+ if (mp->flags & MEMPOOL_F_SC_GET)
+ rg_flags |= RING_F_SC_DEQ;
+
+ /* Allocate the ring that will be used to store objects.
+ * Ring functions will return appropriate errors if we are
+ * running as a secondary process etc., so no checks made
+ * in this function for that condition.
+ */
+ r = rte_ring_create(rg_name, rte_align32pow2(mp->size + 1),
+ mp->socket_id, rg_flags);
+ if (r == NULL)
+ return -rte_errno;
+
+ mp->ring = r;
+ return 0;
+}
+
+/* Free memory chunks used by a mempool. Objects must be in pool */
+static void
+rte_mempool_free_memchunks(struct rte_mempool *mp)
+{
+ struct rte_mempool_memhdr *memhdr;
+ void *elt;
+
+ while (!STAILQ_EMPTY(&mp->elt_list)) {
+ rte_ring_sc_dequeue(mp->ring, &elt);
+ (void)elt;
+ STAILQ_REMOVE_HEAD(&mp->elt_list, next);
+ mp->populated_size--;
+ }
+
+ while (!STAILQ_EMPTY(&mp->mem_list)) {
+ memhdr = STAILQ_FIRST(&mp->mem_list);
+ STAILQ_REMOVE_HEAD(&mp->mem_list, next);
+ rte_free(memhdr);
+ mp->nb_mem_chunks--;
+ }
+}
+
+/* Add objects in the pool, using a physically contiguous memory
+ * zone. Return the number of objects added, or a negative value
+ * on error.
+ */
+static int
+rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr,
+ phys_addr_t paddr, size_t len)
+{
+ unsigned total_elt_sz;
+ unsigned i = 0;
+ size_t off;
+ struct rte_mempool_memhdr *memhdr;
+
+ /* mempool is already populated */
+ if (mp->populated_size >= mp->size)
+ return -ENOSPC;
+
+ total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
+
+ memhdr = rte_zmalloc("MEMPOOL_MEMHDR", sizeof(*memhdr), 0);
+ if (memhdr == NULL)
+ return -ENOMEM;
+
+ memhdr->mp = mp;
+ memhdr->addr = vaddr;
+ memhdr->phys_addr = paddr;
+ memhdr->len = len;
+
+ if (mp->flags & MEMPOOL_F_NO_CACHE_ALIGN)
+ off = RTE_PTR_ALIGN_CEIL(vaddr, 8) - vaddr;
+ else
+ off = RTE_PTR_ALIGN_CEIL(vaddr, RTE_CACHE_LINE_SIZE) - vaddr;
+
+ while (off + total_elt_sz <= len && mp->populated_size < mp->size) {
+ off += mp->header_size;
+ mempool_add_elem(mp, (char *)vaddr + off, paddr + off);
+ off += mp->elt_size + mp->trailer_size;
+ i++;
+ }
+
+ /* not enough room to store one object */
+ if (i == 0)
+ return -EINVAL;
+
+ STAILQ_INSERT_TAIL(&mp->mem_list, memhdr, next);
+ mp->nb_mem_chunks++;
+ return i;
+}
+
+/* Add objects in the pool, using a table of physical pages. Return the
+ * number of objects added, or a negative value on error.
+ */
+static int
+rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr,
+ const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift)
+{
+ uint32_t i, n;
+ int ret, cnt = 0;
+ size_t pg_sz = (size_t)1 << pg_shift;
+
+ /* mempool must not be populated */
+ if (mp->nb_mem_chunks != 0)
+ return -EEXIST;
+
+ for (i = 0; i < pg_num && mp->populated_size < mp->size; i += n) {
+
+ /* populate with the largest group of contiguous pages */
+ for (n = 1; (i + n) < pg_num &&
+ paddr[i] + pg_sz == paddr[i+n]; n++)
+ ;
+
+ ret = rte_mempool_populate_phys(mp, vaddr + i * pg_sz,
+ paddr[i], n * pg_sz);
+ if (ret < 0) {
+ rte_mempool_free_memchunks(mp);
+ return ret;
+ }
+ cnt += ret;
+ }
+ return cnt;
+}
+
/*
* Create the mempool over already allocated chunk of memory.
* That external memory buffer can consists of physically disjoint pages.
const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift)
{
char mz_name[RTE_MEMZONE_NAMESIZE];
- char rg_name[RTE_RING_NAMESIZE];
struct rte_mempool_list *mempool_list;
struct rte_mempool *mp = NULL;
struct rte_tailq_entry *te = NULL;
- struct rte_ring *r = NULL;
const struct rte_memzone *mz;
size_t mempool_size;
int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY;
- int rg_flags = 0;
void *obj;
struct rte_mempool_objsz objsz;
void *startaddr;
int page_size = getpagesize();
+ int ret;
/* compilation-time checks */
RTE_BUILD_BUG_ON((sizeof(struct rte_mempool) &
}
/* Check that pg_num and pg_shift parameters are valid. */
- if (pg_num < RTE_DIM(mp->elt_pa) || pg_shift > MEMPOOL_PG_SHIFT_MAX) {
+ if (pg_num == 0 || pg_shift > MEMPOOL_PG_SHIFT_MAX) {
rte_errno = EINVAL;
return NULL;
}
if (flags & MEMPOOL_F_NO_CACHE_ALIGN)
flags |= MEMPOOL_F_NO_SPREAD;
- /* ring flags */
- if (flags & MEMPOOL_F_SP_PUT)
- rg_flags |= RING_F_SP_ENQ;
- if (flags & MEMPOOL_F_SC_GET)
- rg_flags |= RING_F_SC_DEQ;
-
/* calculate mempool object sizes. */
if (!rte_mempool_calc_obj_size(elt_size, flags, &objsz)) {
rte_errno = EINVAL;
rte_rwlock_write_lock(RTE_EAL_MEMPOOL_RWLOCK);
- /* allocate the ring that will be used to store objects */
- /* Ring functions will return appropriate errors if we are
- * running as a secondary process etc., so no checks made
- * in this function for that condition */
- snprintf(rg_name, sizeof(rg_name), RTE_MEMPOOL_MZ_FORMAT, name);
- r = rte_ring_create(rg_name, rte_align32pow2(n+1), socket_id, rg_flags);
- if (r == NULL)
- goto exit_unlock;
-
/*
* reserve a memory zone for this mempool: private data is
* cache-aligned
* store mempool objects. Otherwise reserve a memzone that is large
* enough to hold mempool header and metadata plus mempool objects.
*/
- mempool_size = MEMPOOL_HEADER_SIZE(mp, pg_num, cache_size);
+ mempool_size = MEMPOOL_HEADER_SIZE(mp, cache_size);
mempool_size += private_data_size;
mempool_size = RTE_ALIGN_CEIL(mempool_size, RTE_MEMPOOL_ALIGN);
if (vaddr == NULL)
memset(mp, 0, sizeof(*mp));
snprintf(mp->name, sizeof(mp->name), "%s", name);
mp->phys_addr = mz->phys_addr;
- mp->ring = r;
+ mp->socket_id = socket_id;
mp->size = n;
mp->flags = flags;
mp->elt_size = objsz.elt_size;
mp->cache_flushthresh = CALC_CACHE_FLUSHTHRESH(cache_size);
mp->private_data_size = private_data_size;
STAILQ_INIT(&mp->elt_list);
+ STAILQ_INIT(&mp->mem_list);
+
+ if (rte_mempool_ring_create(mp) < 0)
+ goto exit_unlock;
/*
* local_cache pointer is set even if cache_size is zero.
* The local_cache points to just past the elt_pa[] array.
*/
mp->local_cache = (struct rte_mempool_cache *)
- RTE_PTR_ADD(mp, MEMPOOL_HEADER_SIZE(mp, pg_num, 0));
-
- /* calculate address of the first element for continuous mempool. */
- obj = (char *)mp + MEMPOOL_HEADER_SIZE(mp, pg_num, cache_size) +
- private_data_size;
- obj = RTE_PTR_ALIGN_CEIL(obj, RTE_MEMPOOL_ALIGN);
+ RTE_PTR_ADD(mp, MEMPOOL_HEADER_SIZE(mp, 0));
- /* populate address translation fields. */
- mp->pg_num = pg_num;
- mp->pg_shift = pg_shift;
- mp->pg_mask = RTE_LEN2MASK(mp->pg_shift, typeof(mp->pg_mask));
+ /* call the initializer */
+ if (mp_init)
+ mp_init(mp, mp_init_arg);
/* mempool elements allocated together with mempool */
if (vaddr == NULL) {
- mp->elt_va_start = (uintptr_t)obj;
- mp->elt_pa[0] = mp->phys_addr +
- (mp->elt_va_start - (uintptr_t)mp);
+ /* calculate address of the first elt for continuous mempool. */
+ obj = (char *)mp + MEMPOOL_HEADER_SIZE(mp, cache_size) +
+ private_data_size;
+ obj = RTE_PTR_ALIGN_CEIL(obj, RTE_MEMPOOL_ALIGN);
+
+ ret = rte_mempool_populate_phys(mp, obj,
+ mp->phys_addr + ((char *)obj - (char *)mp),
+ objsz.total_size * n);
+ if (ret != (int)mp->size)
+ goto exit_unlock;
} else {
- /* mempool elements in a separate chunk of memory. */
- mp->elt_va_start = (uintptr_t)vaddr;
- memcpy(mp->elt_pa, paddr, sizeof (mp->elt_pa[0]) * pg_num);
+ ret = rte_mempool_populate_phys_tab(mp, vaddr,
+ paddr, pg_num, pg_shift);
+ if (ret != (int)mp->size)
+ goto exit_unlock;
}
- mp->elt_va_end = mp->elt_va_start;
-
/* call the initializer */
- if (mp_init)
- mp_init(mp, mp_init_arg);
-
- mempool_populate(mp, n, 1, obj_init, obj_init_arg);
+ if (obj_init)
+ rte_mempool_obj_iter(mp, obj_init, obj_init_arg);
te->data = (void *) mp;
exit_unlock:
rte_rwlock_write_unlock(RTE_EAL_MEMPOOL_RWLOCK);
- rte_ring_free(r);
+ if (mp != NULL) {
+ rte_mempool_free_memchunks(mp);
+ rte_ring_free(mp->ring);
+ }
rte_free(te);
return NULL;
#pragma GCC diagnostic ignored "-Wcast-qual"
#endif
-struct mempool_audit_arg {
- const struct rte_mempool *mp;
- uintptr_t obj_end;
- uint32_t obj_num;
-};
-
/* check and update cookies or panic (internal) */
void rte_mempool_check_cookies(const struct rte_mempool *mp,
void * const *obj_table_const, unsigned n, int free)
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
static void
-mempool_obj_audit(void *arg, void *start, void *end, uint32_t idx)
+mempool_obj_audit(struct rte_mempool *mp, __rte_unused void *opaque,
+ void *obj, __rte_unused unsigned idx)
{
- struct mempool_audit_arg *pa = arg;
- void *obj;
-
- obj = (char *)start + pa->mp->header_size;
- pa->obj_end = (uintptr_t)end;
- pa->obj_num = idx + 1;
- __mempool_check_cookies(pa->mp, &obj, 1, 2);
+ __mempool_check_cookies(mp, &obj, 1, 2);
}
static void
mempool_audit_cookies(struct rte_mempool *mp)
{
- uint32_t elt_sz, num;
- struct mempool_audit_arg arg;
-
- elt_sz = mp->elt_size + mp->header_size + mp->trailer_size;
-
- arg.mp = mp;
- arg.obj_end = mp->elt_va_start;
- arg.obj_num = 0;
-
- num = rte_mempool_obj_iter((void *)mp->elt_va_start,
- mp->size, elt_sz, 1,
- mp->elt_pa, mp->pg_num, mp->pg_shift,
- mempool_obj_audit, &arg);
+ unsigned num;
+ num = rte_mempool_obj_iter(mp, mempool_obj_audit, NULL);
if (num != mp->size) {
- rte_panic("rte_mempool_obj_iter(mempool=%p, size=%u) "
+ rte_panic("rte_mempool_obj_iter(mempool=%p, size=%u) "
"iterated only over %u elements\n",
mp, mp->size, num);
- } else if (arg.obj_end != mp->elt_va_end || arg.obj_num != mp->size) {
- rte_panic("rte_mempool_obj_iter(mempool=%p, size=%u) "
- "last callback va_end: %#tx (%#tx expeceted), "
- "num of objects: %u (%u expected)\n",
- mp, mp->size,
- arg.obj_end, mp->elt_va_end,
- arg.obj_num, mp->size);
}
}
#else
struct rte_mempool_debug_stats sum;
unsigned lcore_id;
#endif
+ struct rte_mempool_memhdr *memhdr;
unsigned common_count;
unsigned cache_count;
+ size_t mem_len = 0;
RTE_ASSERT(f != NULL);
RTE_ASSERT(mp != NULL);
fprintf(f, " flags=%x\n", mp->flags);
fprintf(f, " ring=<%s>@%p\n", mp->ring->name, mp->ring);
fprintf(f, " phys_addr=0x%" PRIx64 "\n", mp->phys_addr);
+ fprintf(f, " nb_mem_chunks=%u\n", mp->nb_mem_chunks);
fprintf(f, " size=%"PRIu32"\n", mp->size);
+ fprintf(f, " populated_size=%"PRIu32"\n", mp->populated_size);
fprintf(f, " header_size=%"PRIu32"\n", mp->header_size);
fprintf(f, " elt_size=%"PRIu32"\n", mp->elt_size);
fprintf(f, " trailer_size=%"PRIu32"\n", mp->trailer_size);
mp->header_size + mp->elt_size + mp->trailer_size);
fprintf(f, " private_data_size=%"PRIu32"\n", mp->private_data_size);
- fprintf(f, " pg_num=%"PRIu32"\n", mp->pg_num);
- fprintf(f, " pg_shift=%"PRIu32"\n", mp->pg_shift);
- fprintf(f, " pg_mask=%#tx\n", mp->pg_mask);
- fprintf(f, " elt_va_start=%#tx\n", mp->elt_va_start);
- fprintf(f, " elt_va_end=%#tx\n", mp->elt_va_end);
- fprintf(f, " elt_pa[0]=0x%" PRIx64 "\n", mp->elt_pa[0]);
-
- if (mp->size != 0)
+
+ STAILQ_FOREACH(memhdr, &mp->mem_list, next)
+ mem_len += memhdr->len;
+ if (mem_len != 0) {
fprintf(f, " avg bytes/object=%#Lf\n",
- (long double)(mp->elt_va_end - mp->elt_va_start) /
- mp->size);
+ (long double)mem_len / mp->size);
+ }
cache_count = rte_mempool_dump_cache(f, mp);
common_count = rte_ring_count(mp->ring);
git.droids-corp.org / dpdk.git / blobdiff