X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_mempool%2Frte_mempool.c;h=0f29e87128e1a83783ae3d32a22e31415a03a6ee;hb=a2b5a8722f20265896cb1ac026c716239f6313bf;hp=03e6b5f732e56a3354bc013ad2b8a1609e849df1;hpb=91ad0349194ad455fe6bba3d8f64bf6aa5dfb48e;p=dpdk.git diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_mempool.c index 03e6b5f732..0f29e87128 100644 --- a/lib/librte_mempool/rte_mempool.c +++ b/lib/librte_mempool/rte_mempool.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "rte_mempool.h" @@ -99,25 +100,44 @@ static unsigned optimize_object_size(unsigned obj_size) return new_obj_size * RTE_MEMPOOL_ALIGN; } +struct pagesz_walk_arg { + int socket_id; + size_t min; +}; + static int find_min_pagesz(const struct rte_memseg_list *msl, void *arg) { - size_t *min = arg; + struct pagesz_walk_arg *wa = arg; + bool valid; + + /* + * we need to only look at page sizes available for a particular socket + * ID. so, we either need an exact match on socket ID (can match both + * native and external memory), or, if SOCKET_ID_ANY was specified as a + * socket ID argument, we must only look at native memory and ignore any + * page sizes associated with external memory. + */ + valid = msl->socket_id == wa->socket_id; + valid |= wa->socket_id == SOCKET_ID_ANY && msl->external == 0; - if (msl->page_sz < *min) - *min = msl->page_sz; + if (valid && msl->page_sz < wa->min) + wa->min = msl->page_sz; return 0; } static size_t -get_min_page_size(void) +get_min_page_size(int socket_id) { - size_t min_pagesz = SIZE_MAX; + struct pagesz_walk_arg wa; + + wa.min = SIZE_MAX; + wa.socket_id = socket_id; - rte_memseg_list_walk(find_min_pagesz, &min_pagesz); + rte_memseg_list_walk(find_min_pagesz, &wa); - return min_pagesz == SIZE_MAX ? (size_t) getpagesize() : min_pagesz; + return wa.min == SIZE_MAX ? (size_t) getpagesize() : wa.min; } @@ -409,7 +429,9 @@ rte_mempool_populate_default(struct rte_mempool *mp) rte_iova_t iova; unsigned mz_id, n; int ret; - bool no_contig, try_contig, no_pageshift; + bool need_iova_contig_obj; + bool try_iova_contig_mempool; + bool alloc_in_ext_mem; ret = mempool_ops_alloc_once(mp); if (ret != 0) @@ -419,8 +441,6 @@ rte_mempool_populate_default(struct rte_mempool *mp) if (mp->nb_mem_chunks != 0) return -EEXIST; - no_contig = mp->flags & MEMPOOL_F_NO_IOVA_CONTIG; - /* * the following section calculates page shift and page size values. * @@ -462,15 +482,32 @@ rte_mempool_populate_default(struct rte_mempool *mp) * in one contiguous chunk as well (otherwise we might end up wasting a * 1G page on a 10MB memzone). If we fail to get enough contiguous * memory, then we'll go and reserve space page-by-page. + * + * We also have to take into account the fact that memory that we're + * going to allocate from can belong to an externally allocated memory + * area, in which case the assumption of IOVA as VA mode being + * synonymous with IOVA contiguousness will not hold. We should also try + * to go for contiguous memory even if we're in no-huge mode, because + * external memory may in fact be IOVA-contiguous. */ - no_pageshift = no_contig || rte_eal_iova_mode() == RTE_IOVA_VA; - try_contig = !no_contig && !no_pageshift && rte_eal_has_hugepages(); - if (no_pageshift) { + /* check if we can retrieve a valid socket ID */ + ret = rte_malloc_heap_socket_is_external(mp->socket_id); + if (ret < 0) + return -EINVAL; + alloc_in_ext_mem = (ret == 1); + need_iova_contig_obj = !(mp->flags & MEMPOOL_F_NO_IOVA_CONTIG); + try_iova_contig_mempool = false; + + if (!need_iova_contig_obj) { + pg_sz = 0; + pg_shift = 0; + } else if (!alloc_in_ext_mem && rte_eal_iova_mode() == RTE_IOVA_VA) { pg_sz = 0; pg_shift = 0; - } else if (try_contig) { - pg_sz = get_min_page_size(); + } else if (rte_eal_has_hugepages() || alloc_in_ext_mem) { + try_iova_contig_mempool = true; + pg_sz = get_min_page_size(mp->socket_id); pg_shift = rte_bsf32(pg_sz); } else { pg_sz = getpagesize(); @@ -481,7 +518,7 @@ rte_mempool_populate_default(struct rte_mempool *mp) size_t min_chunk_size; unsigned int flags; - if (try_contig || no_pageshift) + if (try_iova_contig_mempool || pg_sz == 0) mem_size = rte_mempool_ops_calc_mem_size(mp, n, 0, &min_chunk_size, &align); else @@ -505,7 +542,7 @@ rte_mempool_populate_default(struct rte_mempool *mp) /* if we're trying to reserve contiguous memory, add appropriate * memzone flag. */ - if (try_contig) + if (try_iova_contig_mempool) flags |= RTE_MEMZONE_IOVA_CONTIG; mz = rte_memzone_reserve_aligned(mz_name, mem_size, @@ -515,8 +552,9 @@ rte_mempool_populate_default(struct rte_mempool *mp) * minimum required contiguous chunk fits minimum page, adjust * memzone size to the page size, and try again. */ - if (mz == NULL && try_contig && min_chunk_size <= pg_sz) { - try_contig = false; + if (mz == NULL && try_iova_contig_mempool && + min_chunk_size <= pg_sz) { + try_iova_contig_mempool = false; flags &= ~RTE_MEMZONE_IOVA_CONTIG; mem_size = rte_mempool_ops_calc_mem_size(mp, n, @@ -551,12 +589,12 @@ rte_mempool_populate_default(struct rte_mempool *mp) goto fail; } - if (no_contig) - iova = RTE_BAD_IOVA; - else + if (need_iova_contig_obj) iova = mz->iova; + else + iova = RTE_BAD_IOVA; - if (no_pageshift || try_contig) + if (try_iova_contig_mempool || pg_sz == 0) ret = rte_mempool_populate_iova(mp, mz->addr, iova, mz->len, rte_mempool_memchunk_mz_free, @@ -676,7 +714,7 @@ rte_mempool_free(struct rte_mempool *mp) return; mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); - rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); + rte_mcfg_tailq_write_lock(); /* find out tailq entry */ TAILQ_FOREACH(te, mempool_list, next) { if (te->data == (void *)mp) @@ -687,7 +725,7 @@ rte_mempool_free(struct rte_mempool *mp) TAILQ_REMOVE(mempool_list, te, next); rte_free(te); } - rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + rte_mcfg_tailq_write_unlock(); rte_mempool_free_memchunks(mp); rte_mempool_ops_free(mp); @@ -795,7 +833,7 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, return NULL; } - rte_rwlock_write_lock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_write_lock(); /* * reserve a memory zone for this mempool: private data is @@ -829,7 +867,7 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, /* init the mempool structure */ mp = mz->addr; memset(mp, 0, MEMPOOL_HEADER_SIZE(mp, cache_size)); - ret = snprintf(mp->name, sizeof(mp->name), "%s", name); + ret = strlcpy(mp->name, name, sizeof(mp->name)); if (ret < 0 || ret >= (int)sizeof(mp->name)) { rte_errno = ENAMETOOLONG; goto exit_unlock; @@ -863,15 +901,15 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, te->data = mp; - rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); + rte_mcfg_tailq_write_lock(); TAILQ_INSERT_TAIL(mempool_list, te, next); - rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); - rte_rwlock_write_unlock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_tailq_write_unlock(); + rte_mcfg_mempool_write_unlock(); return mp; exit_unlock: - rte_rwlock_write_unlock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_write_unlock(); rte_free(te); rte_mempool_free(mp); return NULL; @@ -1233,14 +1271,14 @@ rte_mempool_list_dump(FILE *f) mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); - rte_rwlock_read_lock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_read_lock(); TAILQ_FOREACH(te, mempool_list, next) { mp = (struct rte_mempool *) te->data; rte_mempool_dump(f, mp); } - rte_rwlock_read_unlock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_read_unlock(); } /* search a mempool from its name */ @@ -1253,7 +1291,7 @@ rte_mempool_lookup(const char *name) mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); - rte_rwlock_read_lock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_read_lock(); TAILQ_FOREACH(te, mempool_list, next) { mp = (struct rte_mempool *) te->data; @@ -1261,7 +1299,7 @@ rte_mempool_lookup(const char *name) break; } - rte_rwlock_read_unlock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_read_unlock(); if (te == NULL) { rte_errno = ENOENT; @@ -1280,11 +1318,11 @@ void rte_mempool_walk(void (*func)(struct rte_mempool *, void *), mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); - rte_rwlock_read_lock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_read_lock(); TAILQ_FOREACH_SAFE(te, mempool_list, next, tmp_te) { (*func)((struct rte_mempool *) te->data, arg); } - rte_rwlock_read_unlock(RTE_EAL_MEMPOOL_RWLOCK); + rte_mcfg_mempool_read_unlock(); }