X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_mempool%2Frte_mempool.c;h=1ab670118bcbe2dc4a35f54a54d257a1f9b8c603;hb=ce94a51ff05c0a4b63177f8a314feb5d19992036;hp=33af51b9d49d8da4c2f3bcbf180b2ee97106021a;hpb=85226f9c526bc15d203e5be40bc4ea1f7bf390ef;p=dpdk.git diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_mempool.c index 33af51b9d4..1ab670118b 100644 --- a/lib/librte_mempool/rte_mempool.c +++ b/lib/librte_mempool/rte_mempool.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -299,35 +300,18 @@ rte_mempool_xmem_usage(__rte_unused void *vaddr, uint32_t elt_num, return (size_t)paddr_idx << pg_shift; } -#ifndef RTE_LIBRTE_XEN_DOM0 -/* stub if DOM0 support not configured */ -struct rte_mempool * -rte_dom0_mempool_create(const char *name __rte_unused, - unsigned n __rte_unused, - unsigned elt_size __rte_unused, - unsigned cache_size __rte_unused, - unsigned private_data_size __rte_unused, - rte_mempool_ctor_t *mp_init __rte_unused, - void *mp_init_arg __rte_unused, - rte_mempool_obj_ctor_t *obj_init __rte_unused, - void *obj_init_arg __rte_unused, - int socket_id __rte_unused, - unsigned flags __rte_unused) -{ - rte_errno = EINVAL; - return NULL; -} -#endif - /* create the internal ring */ static int rte_mempool_ring_create(struct rte_mempool *mp) { - int rg_flags = 0; + int rg_flags = 0, ret; char rg_name[RTE_RING_NAMESIZE]; struct rte_ring *r; - snprintf(rg_name, sizeof(rg_name), RTE_MEMPOOL_MZ_FORMAT, mp->name); + ret = snprintf(rg_name, sizeof(rg_name), + RTE_MEMPOOL_MZ_FORMAT, mp->name); + if (ret < 0 || ret >= (int)sizeof(rg_name)) + return -ENAMETOOLONG; /* ring flags */ if (mp->flags & MEMPOOL_F_SP_PUT) @@ -346,6 +330,7 @@ rte_mempool_ring_create(struct rte_mempool *mp) return -rte_errno; mp->ring = r; + mp->flags |= MEMPOOL_F_RING_CREATED; return 0; } @@ -386,7 +371,7 @@ rte_mempool_free_memchunks(struct rte_mempool *mp) * zone. Return the number of objects added, or a negative value * on error. */ -static int +int rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) @@ -395,6 +380,14 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, unsigned i = 0; size_t off; struct rte_mempool_memhdr *memhdr; + int ret; + + /* create the internal ring if not already done */ + if ((mp->flags & MEMPOOL_F_RING_CREATED) == 0) { + ret = rte_mempool_ring_create(mp); + if (ret < 0) + return ret; + } /* mempool is already populated */ if (mp->populated_size >= mp->size) @@ -420,7 +413,11 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, while (off + total_elt_sz <= len && mp->populated_size < mp->size) { off += mp->header_size; - mempool_add_elem(mp, (char *)vaddr + off, paddr + off); + if (paddr == RTE_BAD_PHYS_ADDR) + mempool_add_elem(mp, (char *)vaddr + off, + RTE_BAD_PHYS_ADDR); + else + mempool_add_elem(mp, (char *)vaddr + off, paddr + off); off += mp->elt_size + mp->trailer_size; i++; } @@ -437,7 +434,7 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, /* Add objects in the pool, using a table of physical pages. Return the * number of objects added, or a negative value on error. */ -static int +int rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) @@ -450,6 +447,10 @@ rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, if (mp->nb_mem_chunks != 0) return -EEXIST; + if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) + return rte_mempool_populate_phys(mp, vaddr, RTE_BAD_PHYS_ADDR, + pg_num * pg_sz, free_cb, opaque); + for (i = 0; i < pg_num && mp->populated_size < mp->size; i += n) { /* populate with the largest group of contiguous pages */ @@ -473,7 +474,7 @@ rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, /* Populate the mempool with a virtual area. Return the number of * objects added, or a negative value on error. */ -static int +int rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) @@ -491,10 +492,17 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, if (RTE_ALIGN_CEIL(len, pg_sz) != len) return -EINVAL; + if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) + return rte_mempool_populate_phys(mp, addr, RTE_BAD_PHYS_ADDR, + len, free_cb, opaque); + for (off = 0; off + pg_sz <= len && mp->populated_size < mp->size; off += phys_len) { paddr = rte_mem_virt2phy(addr + off); + /* required for xen_dom0 to get the machine address */ + paddr = rte_mem_phy2mch(-1, paddr); + if (paddr == RTE_BAD_PHYS_ADDR) { ret = -EINVAL; goto fail; @@ -531,13 +539,14 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, * and populate them. Return the number of objects added, or a negative * value on error. */ -static int +int rte_mempool_populate_default(struct rte_mempool *mp) { int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; char mz_name[RTE_MEMZONE_NAMESIZE]; const struct rte_memzone *mz; size_t size, total_elt_sz, align, pg_sz, pg_shift; + phys_addr_t paddr; unsigned mz_id, n; int ret; @@ -577,9 +586,14 @@ rte_mempool_populate_default(struct rte_mempool *mp) goto fail; } - if (rte_eal_has_hugepages()) + if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) + paddr = RTE_BAD_PHYS_ADDR; + else + paddr = mz->phys_addr; + + if (rte_eal_has_hugepages() && !rte_xen_dom0_supported()) ret = rte_mempool_populate_phys(mp, mz->addr, - mz->phys_addr, mz->len, + paddr, mz->len, rte_mempool_memchunk_mz_free, (void *)(uintptr_t)mz); else @@ -598,8 +612,71 @@ rte_mempool_populate_default(struct rte_mempool *mp) return ret; } -/* free a mempool */ +/* return the memory size required for mempool objects in anonymous mem */ +static size_t +get_anon_size(const struct rte_mempool *mp) +{ + size_t size, total_elt_sz, pg_sz, pg_shift; + + pg_sz = getpagesize(); + pg_shift = rte_bsf32(pg_sz); + total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; + size = rte_mempool_xmem_size(mp->size, total_elt_sz, pg_shift); + + return size; +} + +/* unmap a memory zone mapped by rte_mempool_populate_anon() */ static void +rte_mempool_memchunk_anon_free(struct rte_mempool_memhdr *memhdr, + void *opaque) +{ + munmap(opaque, get_anon_size(memhdr->mp)); +} + +/* populate the mempool with an anonymous mapping */ +int +rte_mempool_populate_anon(struct rte_mempool *mp) +{ + size_t size; + int ret; + char *addr; + + /* mempool is already populated, error */ + if (!STAILQ_EMPTY(&mp->mem_list)) { + rte_errno = EINVAL; + return 0; + } + + /* get chunk of virtually continuous memory */ + size = get_anon_size(mp); + addr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) { + rte_errno = errno; + return 0; + } + /* can't use MMAP_LOCKED, it does not exist on BSD */ + if (mlock(addr, size) < 0) { + rte_errno = errno; + munmap(addr, size); + return 0; + } + + ret = rte_mempool_populate_virt(mp, addr, size, getpagesize(), + rte_mempool_memchunk_anon_free, addr); + if (ret == 0) + goto fail; + + return mp->populated_size; + + fail: + rte_mempool_free_memchunks(mp); + return 0; +} + +/* free a mempool */ +void rte_mempool_free(struct rte_mempool *mp) { struct rte_mempool_list *mempool_list = NULL; @@ -628,7 +705,7 @@ rte_mempool_free(struct rte_mempool *mp) } /* create an empty mempool */ -static struct rte_mempool * +struct rte_mempool * rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, int socket_id, unsigned flags) @@ -641,6 +718,7 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, size_t mempool_size; int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; struct rte_mempool_objsz objsz; + int ret; /* compilation-time checks */ RTE_BUILD_BUG_ON((sizeof(struct rte_mempool) & @@ -694,7 +772,11 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, mempool_size += private_data_size; mempool_size = RTE_ALIGN_CEIL(mempool_size, RTE_MEMPOOL_ALIGN); - snprintf(mz_name, sizeof(mz_name), RTE_MEMPOOL_MZ_FORMAT, name); + ret = snprintf(mz_name, sizeof(mz_name), RTE_MEMPOOL_MZ_FORMAT, name); + if (ret < 0 || ret >= (int)sizeof(mz_name)) { + rte_errno = ENAMETOOLONG; + goto exit_unlock; + } mz = rte_memzone_reserve(mz_name, mempool_size, socket_id, mz_flags); if (mz == NULL) @@ -703,7 +785,11 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, /* init the mempool structure */ mp = mz->addr; memset(mp, 0, sizeof(*mp)); - snprintf(mp->name, sizeof(mp->name), "%s", name); + ret = snprintf(mp->name, sizeof(mp->name), "%s", name); + if (ret < 0 || ret >= (int)sizeof(mp->name)) { + rte_errno = ENAMETOOLONG; + goto exit_unlock; + } mp->mz = mz; mp->socket_id = socket_id; mp->size = n; @@ -718,9 +804,6 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, STAILQ_INIT(&mp->elt_list); STAILQ_INIT(&mp->mem_list); - if (rte_mempool_ring_create(mp) < 0) - goto exit_unlock; - /* * local_cache pointer is set even if cache_size is zero. * The local_cache points to just past the elt_pa[] array. @@ -753,13 +836,6 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size, { struct rte_mempool *mp; - if (rte_xen_dom0_supported()) - return rte_dom0_mempool_create(name, n, elt_size, - cache_size, private_data_size, - mp_init, mp_init_arg, - obj_init, obj_init_arg, - socket_id, flags); - mp = rte_mempool_create_empty(name, n, elt_size, cache_size, private_data_size, socket_id, flags); if (mp == NULL)