X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_mempool%2Frte_mempool.h;h=9745bf0d70b4a9e238f68652aca01f78c7ba6dad;hb=693f715da45c48ec1ec0fe4ba2f3b5ffd11ba53e;hp=95f19f9f73ca40975ced0ba95b9855b417869106;hpb=dd0024ccbc7064a104614e708a9f885a9a741a2e;p=dpdk.git diff --git a/lib/librte_mempool/rte_mempool.h b/lib/librte_mempool/rte_mempool.h index 95f19f9f73..9745bf0d70 100644 --- a/lib/librte_mempool/rte_mempool.h +++ b/lib/librte_mempool/rte_mempool.h @@ -109,6 +109,9 @@ struct rte_mempool_cache { } __rte_cache_aligned; #endif /* RTE_MEMPOOL_CACHE_MAX_SIZE > 0 */ +/** + * A structure that stores the size of mempool elements. + */ struct rte_mempool_objsz { uint32_t elt_size; /**< Size of an element. */ uint32_t header_size; /**< Size of header (before elt). */ @@ -139,6 +142,39 @@ struct rte_mempool_objsz { /** Mempool over one chunk of physically continuous memory */ #define MEMPOOL_PG_NUM_DEFAULT 1 +#ifndef RTE_MEMPOOL_ALIGN +#define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE +#endif + +#define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1) + +/** + * Mempool object header structure + * + * Each object stored in mempools are prefixed by this header structure, + * it allows to retrieve the mempool pointer from the object. When debug + * is enabled, a cookie is also added in this structure preventing + * corruptions and double-frees. + */ +struct rte_mempool_objhdr { + struct rte_mempool *mp; /**< The mempool owning the object. */ +#ifdef RTE_LIBRTE_MEMPOOL_DEBUG + uint64_t cookie; /**< Debug cookie. */ +#endif +}; + +/** + * Mempool object trailer structure + * + * In debug mode, each object stored in mempools are suffixed by this + * trailer structure containing a cookie preventing memory corruptions. + */ +struct rte_mempool_objtlr { +#ifdef RTE_LIBRTE_MEMPOOL_DEBUG + uint64_t cookie; /**< Debug cookie. */ +#endif +}; + /** * The RTE mempool structure. */ @@ -179,7 +215,7 @@ struct rte_mempool { uintptr_t elt_va_end; /**< Virtual address of the mempool object. */ phys_addr_t elt_pa[MEMPOOL_PG_NUM_DEFAULT]; - /**< Array of physical pages addresses for the mempool objects buffer. */ + /**< Array of physical page addresses for the mempool objects buffer. */ } __rte_cache_aligned; @@ -190,6 +226,7 @@ struct rte_mempool { /** * @internal When debug is enabled, store some statistics. + * * @param mp * Pointer to the memory pool. * @param name @@ -198,51 +235,40 @@ struct rte_mempool { * Number to add to the object-oriented statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG -#define __MEMPOOL_STAT_ADD(mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - mp->stats[__lcore_id].name##_objs += n; \ - mp->stats[__lcore_id].name##_bulk += 1; \ +#define __MEMPOOL_STAT_ADD(mp, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) { \ + mp->stats[__lcore_id].name##_objs += n; \ + mp->stats[__lcore_id].name##_bulk += 1; \ + } \ } while(0) #else #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0) #endif /** - * Calculates size of the mempool header. + * Calculate the size of the mempool header. + * * @param mp * Pointer to the memory pool. * @param pgn - * Number of page used to store mempool objects. + * Number of pages used to store mempool objects. */ #define MEMPOOL_HEADER_SIZE(mp, pgn) (sizeof(*(mp)) + \ RTE_ALIGN_CEIL(((pgn) - RTE_DIM((mp)->elt_pa)) * \ - sizeof ((mp)->elt_pa[0]), CACHE_LINE_SIZE)) + sizeof ((mp)->elt_pa[0]), RTE_CACHE_LINE_SIZE)) /** - * Returns TRUE if whole mempool is allocated in one contiguous block of memory. + * Return true if the whole mempool is in contiguous memory. */ #define MEMPOOL_IS_CONTIG(mp) \ ((mp)->pg_num == MEMPOOL_PG_NUM_DEFAULT && \ (mp)->phys_addr == (mp)->elt_pa[0]) -/** - * @internal Get a pointer to a mempool pointer in the object header. - * @param obj - * Pointer to object. - * @return - * The pointer to the mempool from which the object was allocated. - */ -static inline struct rte_mempool **__mempool_from_obj(void *obj) +/* return the header of a mempool object (internal) */ +static inline struct rte_mempool_objhdr *__mempool_get_header(void *obj) { - struct rte_mempool **mpp; - unsigned off; - - off = sizeof(struct rte_mempool *); -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG - off += sizeof(uint64_t); -#endif - mpp = (struct rte_mempool **)((char *)obj - off); - return mpp; + return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj, sizeof(struct rte_mempool_objhdr)); } /** @@ -254,49 +280,19 @@ static inline struct rte_mempool **__mempool_from_obj(void *obj) * @return * A pointer to the mempool structure. */ -static inline const struct rte_mempool *rte_mempool_from_obj(void *obj) +static inline struct rte_mempool *rte_mempool_from_obj(void *obj) { - struct rte_mempool * const *mpp; - mpp = __mempool_from_obj(obj); - return *mpp; + struct rte_mempool_objhdr *hdr = __mempool_get_header(obj); + return hdr->mp; } -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG -/* get header cookie value */ -static inline uint64_t __mempool_read_header_cookie(const void *obj) +/* return the trailer of a mempool object (internal) */ +static inline struct rte_mempool_objtlr *__mempool_get_trailer(void *obj) { - return *(const uint64_t *)((const char *)obj - sizeof(uint64_t)); + struct rte_mempool *mp = rte_mempool_from_obj(obj); + return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size); } -/* get trailer cookie value */ -static inline uint64_t __mempool_read_trailer_cookie(void *obj) -{ - struct rte_mempool **mpp = __mempool_from_obj(obj); - return *(uint64_t *)((char *)obj + (*mpp)->elt_size); -} - -/* write header cookie value */ -static inline void __mempool_write_header_cookie(void *obj, int free) -{ - uint64_t *cookie_p; - cookie_p = (uint64_t *)((char *)obj - sizeof(uint64_t)); - if (free == 0) - *cookie_p = RTE_MEMPOOL_HEADER_COOKIE1; - else - *cookie_p = RTE_MEMPOOL_HEADER_COOKIE2; - -} - -/* write trailer cookie value */ -static inline void __mempool_write_trailer_cookie(void *obj) -{ - uint64_t *cookie_p; - struct rte_mempool **mpp = __mempool_from_obj(obj); - cookie_p = (uint64_t *)((char *)obj + (*mpp)->elt_size); - *cookie_p = RTE_MEMPOOL_TRAILER_COOKIE; -} -#endif /* RTE_LIBRTE_MEMPOOL_DEBUG */ - /** * @internal Check and update cookies or panic. * @@ -313,13 +309,14 @@ static inline void __mempool_write_trailer_cookie(void *obj) */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG #ifndef __INTEL_COMPILER -#pragma GCC push_options #pragma GCC diagnostic ignored "-Wcast-qual" #endif static inline void __mempool_check_cookies(const struct rte_mempool *mp, void * const *obj_table_const, unsigned n, int free) { + struct rte_mempool_objhdr *hdr; + struct rte_mempool_objtlr *tlr; uint64_t cookie; void *tmp; void *obj; @@ -337,79 +334,87 @@ static inline void __mempool_check_cookies(const struct rte_mempool *mp, rte_panic("MEMPOOL: object is owned by another " "mempool\n"); - cookie = __mempool_read_header_cookie(obj); + hdr = __mempool_get_header(obj); + cookie = hdr->cookie; if (free == 0) { if (cookie != RTE_MEMPOOL_HEADER_COOKIE1) { rte_log_set_history(0); RTE_LOG(CRIT, MEMPOOL, - "obj=%p, mempool=%p, cookie=%"PRIx64"\n", - obj, mp, cookie); + "obj=%p, mempool=%p, cookie=%" PRIx64 "\n", + obj, (const void *) mp, cookie); rte_panic("MEMPOOL: bad header cookie (put)\n"); } - __mempool_write_header_cookie(obj, 1); + hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE2; } else if (free == 1) { if (cookie != RTE_MEMPOOL_HEADER_COOKIE2) { rte_log_set_history(0); RTE_LOG(CRIT, MEMPOOL, - "obj=%p, mempool=%p, cookie=%"PRIx64"\n", - obj, mp, cookie); + "obj=%p, mempool=%p, cookie=%" PRIx64 "\n", + obj, (const void *) mp, cookie); rte_panic("MEMPOOL: bad header cookie (get)\n"); } - __mempool_write_header_cookie(obj, 0); + hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE1; } else if (free == 2) { if (cookie != RTE_MEMPOOL_HEADER_COOKIE1 && cookie != RTE_MEMPOOL_HEADER_COOKIE2) { rte_log_set_history(0); RTE_LOG(CRIT, MEMPOOL, - "obj=%p, mempool=%p, cookie=%"PRIx64"\n", - obj, mp, cookie); + "obj=%p, mempool=%p, cookie=%" PRIx64 "\n", + obj, (const void *) mp, cookie); rte_panic("MEMPOOL: bad header cookie (audit)\n"); } } - cookie = __mempool_read_trailer_cookie(obj); + tlr = __mempool_get_trailer(obj); + cookie = tlr->cookie; if (cookie != RTE_MEMPOOL_TRAILER_COOKIE) { rte_log_set_history(0); RTE_LOG(CRIT, MEMPOOL, - "obj=%p, mempool=%p, cookie=%"PRIx64"\n", - obj, mp, cookie); + "obj=%p, mempool=%p, cookie=%" PRIx64 "\n", + obj, (const void *) mp, cookie); rte_panic("MEMPOOL: bad trailer cookie\n"); } } } #ifndef __INTEL_COMPILER -#pragma GCC pop_options +#pragma GCC diagnostic error "-Wcast-qual" #endif #else #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0) #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */ /** - * An mempool's object iterator callback function. + * A mempool object iterator callback function. */ typedef void (*rte_mempool_obj_iter_t)(void * /*obj_iter_arg*/, void * /*obj_start*/, void * /*obj_end*/, uint32_t /*obj_index */); -/* - * Iterates across objects of the given size and alignment in the +/** + * Call a function for each mempool object in a memory chunk + * + * Iterate across objects of the given size and alignment in the * provided chunk of memory. The given memory buffer can consist of - * disjoint physical pages. - * For each object calls the provided callback (if any). - * Used to populate mempool, walk through all elements of the mempool, - * estimate how many elements of the given size could be created in the given - * memory buffer. + * disjointed physical pages. + * + * For each object, call the provided callback (if any). This function + * is used to populate a mempool, or walk through all the elements of a + * mempool, or estimate how many elements of the given size could be + * created in the given memory buffer. + * * @param vaddr * Virtual address of the memory buffer. * @param elt_num * Maximum number of objects to iterate through. * @param elt_sz * Size of each object. + * @param align + * Alignment of each object. * @param paddr - * Array of phyiscall addresses of the pages that comprises given memory + * Array of physical addresses of the pages that comprises given memory * buffer. * @param pg_num * Number of elements in the paddr array. @@ -418,12 +423,11 @@ typedef void (*rte_mempool_obj_iter_t)(void * /*obj_iter_arg*/, * @param obj_iter * Object iterator callback function (could be NULL). * @param obj_iter_arg - * User defined Prameter for the object iterator callback function. + * User defined parameter for the object iterator callback function. * * @return * Number of objects iterated through. */ - uint32_t rte_mempool_obj_iter(void *vaddr, uint32_t elt_num, size_t elt_sz, size_t align, const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, @@ -448,7 +452,7 @@ typedef void (rte_mempool_obj_ctor_t)(struct rte_mempool *, void *, typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *); /** - * Creates a new mempool named *name* in memory. + * Create a new mempool named *name* in memory. * * This function uses ``memzone_reserve()`` to allocate memory. The * pool contains n elements of elt_size. Its size is set to n. @@ -467,7 +471,7 @@ typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *); * If cache_size is non-zero, the rte_mempool library will try to * limit the accesses to the common lockless pool, by maintaining a * per-lcore object cache. This argument must be lower or equal to - * CONFIG_RTE_MEMPOOL_CACHE_MAX_SIZE. It is advised to choose + * CONFIG_RTE_MEMPOOL_CACHE_MAX_SIZE and n / 1.5. It is advised to choose * cache_size to have "n modulo cache_size == 0": if this is * not the case, some elements will always stay in the pool and will * never be used. The access to the per-lcore table is of course @@ -523,7 +527,6 @@ typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *); * with rte_errno set appropriately. Possible rte_errno values include: * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure * - E_RTE_SECONDARY - function was called from a secondary process instance - * - E_RTE_NO_TAILQ - no tailq list could be got for the ring or mempool list * - EINVAL - cache size provided is too large * - ENOSPC - the maximum number of memzones has already been allocated * - EEXIST - a memzone with the same name already exists @@ -537,14 +540,14 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size, int socket_id, unsigned flags); /** - * Creates a new mempool named *name* in memory. + * Create a new mempool named *name* in memory. * * This function uses ``memzone_reserve()`` to allocate memory. The * pool contains n elements of elt_size. Its size is set to n. * Depending on the input parameters, mempool elements can be either allocated * together with the mempool header, or an externally provided memory buffer * could be used to store mempool objects. In later case, that external - * memory buffer can consist of set of disjoint phyiscal pages. + * memory buffer can consist of set of disjoint physical pages. * * @param name * The name of the mempool. @@ -613,7 +616,7 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size, * Virtual address of the externally allocated memory buffer. * Will be used to store mempool objects. * @param paddr - * Array of phyiscall addresses of the pages that comprises given memory + * Array of physical addresses of the pages that comprises given memory * buffer. * @param pg_num * Number of elements in the paddr array. @@ -624,7 +627,6 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size, * with rte_errno set appropriately. Possible rte_errno values include: * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure * - E_RTE_SECONDARY - function was called from a secondary process instance - * - E_RTE_NO_TAILQ - no tailq list could be got for the ring or mempool list * - EINVAL - cache size provided is too large * - ENOSPC - the maximum number of memzones has already been allocated * - EEXIST - a memzone with the same name already exists @@ -638,14 +640,13 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, int socket_id, unsigned flags, void *vaddr, const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift); -#ifdef RTE_LIBRTE_XEN_DOM0 /** - * Creates a new mempool named *name* in memory on Xen Dom0. + * Create a new mempool named *name* in memory on Xen Dom0. * * This function uses ``rte_mempool_xmem_create()`` to allocate memory. The * pool contains n elements of elt_size. Its size is set to n. * All elements of the mempool are allocated together with the mempool header, - * and memory buffer can consist of set of disjoint phyiscal pages. + * and memory buffer can consist of set of disjoint physical pages. * * @param name * The name of the mempool. @@ -715,7 +716,6 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, * with rte_errno set appropriately. Possible rte_errno values include: * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure * - E_RTE_SECONDARY - function was called from a secondary process instance - * - E_RTE_NO_TAILQ - no tailq list could be got for the ring or mempool list * - EINVAL - cache size provided is too large * - ENOSPC - the maximum number of memzones has already been allocated * - EEXIST - a memzone with the same name already exists @@ -727,7 +727,7 @@ rte_dom0_mempool_create(const char *name, unsigned n, unsigned elt_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags); -#endif + /** * Dump the status of the mempool to the console. @@ -768,8 +768,9 @@ __mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table, __MEMPOOL_STAT_ADD(mp, put, n); #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 - /* cache is not enabled or single producer */ - if (unlikely(cache_size == 0 || is_mp == 0)) + /* cache is not enabled or single producer or non-EAL thread */ + if (unlikely(cache_size == 0 || is_mp == 0 || + lcore_id >= RTE_MAX_LCORE)) goto ring_enqueue; /* Go straight to ring if put would overflow mem allocated for cache */ @@ -945,9 +946,6 @@ __mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n, int is_mc) { int ret; -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG - unsigned n_orig = n; -#endif #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0 struct rte_mempool_cache *cache; uint32_t index, len; @@ -956,7 +954,8 @@ __mempool_get_bulk(struct rte_mempool *mp, void **obj_table, uint32_t cache_size = mp->cache_size; /* cache is not enabled or single consumer */ - if (unlikely(cache_size == 0 || is_mc == 0 || n >= cache_size)) + if (unlikely(cache_size == 0 || is_mc == 0 || + n >= cache_size || lcore_id >= RTE_MAX_LCORE)) goto ring_dequeue; cache = &mp->local_cache[lcore_id]; @@ -988,7 +987,7 @@ __mempool_get_bulk(struct rte_mempool *mp, void **obj_table, cache->len -= n; - __MEMPOOL_STAT_ADD(mp, get_success, n_orig); + __MEMPOOL_STAT_ADD(mp, get_success, n); return 0; @@ -1002,9 +1001,9 @@ ring_dequeue: ret = rte_ring_sc_dequeue_bulk(mp->ring, obj_table, n); if (ret < 0) - __MEMPOOL_STAT_ADD(mp, get_fail, n_orig); + __MEMPOOL_STAT_ADD(mp, get_fail, n); else - __MEMPOOL_STAT_ADD(mp, get_success, n_orig); + __MEMPOOL_STAT_ADD(mp, get_success, n); return ret; } @@ -1261,7 +1260,7 @@ rte_mempool_virt2phy(const struct rte_mempool *mp, const void *elt) uintptr_t off; off = (const char *)elt - (const char *)mp->elt_va_start; - return (mp->elt_pa[off >> mp->pg_shift] + (off & mp->pg_mask)); + return mp->elt_pa[off >> mp->pg_shift] + (off & mp->pg_mask); } else { /* * If huge pages are disabled, we cannot assume the @@ -1320,14 +1319,19 @@ void rte_mempool_list_dump(FILE *f); struct rte_mempool *rte_mempool_lookup(const char *name); /** + * Get the header, trailer and total size of a mempool element. + * * Given a desired size of the mempool element and mempool flags, - * caluclates header, trailer, body and total sizes of the mempool object. + * calculates header, trailer, body and total sizes of the mempool object. + * * @param elt_size * The size of each element. * @param flags * The flags used for the mempool creation. * Consult rte_mempool_create() for more information about possible values. * The size of each element. + * @param sz + * The calculated detailed size the mempool object. May be NULL. * @return * Total size of the mempool object. */ @@ -1335,11 +1339,16 @@ uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct rte_mempool_objsz *sz); /** - * Calculate maximum amount of memory required to store given number of objects. - * Assumes that the memory buffer will be aligned at page boundary. - * Note, that if object size is bigger then page size, then it assumes that - * we have a subsets of physically continuous pages big enough to store - * at least one object. + * Get the size of memory required to store mempool elements. + * + * Calculate the maximum amount of memory required to store given number + * of objects. Assume that the memory buffer will be aligned at page + * boundary. + * + * Note that if object size is bigger then page size, then it assumes + * that pages are grouped in subsets of physically continuous pages big + * enough to store at least one object. + * * @param elt_num * Number of elements. * @param elt_sz @@ -1353,8 +1362,11 @@ size_t rte_mempool_xmem_size(uint32_t elt_num, size_t elt_sz, uint32_t pg_shift); /** + * Get the size of memory required to store mempool elements. + * * Calculate how much memory would be actually required with the given * memory footprint to store required number of objects. + * * @param vaddr * Virtual address of the externally allocated memory buffer. * Will be used to store mempool objects. @@ -1363,17 +1375,17 @@ size_t rte_mempool_xmem_size(uint32_t elt_num, size_t elt_sz, * @param elt_sz * The size of each element. * @param paddr - * Array of phyiscall addresses of the pages that comprises given memory + * Array of physical addresses of the pages that comprises given memory * buffer. * @param pg_num * Number of elements in the paddr array. * @param pg_shift * LOG2 of the physical pages size. * @return - * Number of bytes needed to store given number of objects, - * aligned to the given page size. - * If provided memory buffer is not big enough: - * (-1) * actual number of elemnts that can be stored in that buffer. + * On success, the number of bytes needed to store given number of + * objects, aligned to the given page size. If the provided memory + * buffer is too small, return a negative value whose absolute value + * is the actual number of elements that can be stored in that buffer. */ ssize_t rte_mempool_xmem_usage(void *vaddr, uint32_t elt_num, size_t elt_sz, const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift);