X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_ring%2Frte_ring.h;h=f8ac7f5d91587b4722072057961b373b62f9870f;hb=82cb88375cfb8ec0ab23e5f663890bc14469f25b;hp=da54e3431c99957869c726928fd13bab973e8dbe;hpb=591a9d7985c1230652d9f7ea1f9221e8c66ec188;p=dpdk.git diff --git a/lib/librte_ring/rte_ring.h b/lib/librte_ring/rte_ring.h index da54e3431c..f8ac7f5d91 100644 --- a/lib/librte_ring/rte_ring.h +++ b/lib/librte_ring/rte_ring.h @@ -1,13 +1,13 @@ /*- * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. - * + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: - * + * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright @@ -17,7 +17,7 @@ * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -100,32 +100,36 @@ extern "C" { #include #include #include +#include + +#define RTE_TAILQ_RING_NAME "RTE_RING" enum rte_ring_queue_behavior { RTE_RING_QUEUE_FIXED = 0, /* Enq/Deq a fixed number of items from a ring */ - RTE_RING_QUEUE_VARIABLE /* Enq/Deq as many items a possible from ring */ + RTE_RING_QUEUE_VARIABLE /* Enq/Deq as many items as possible from ring */ }; -#ifdef RTE_LIBRTE_RING_DEBUG -/** - * A structure that stores the ring statistics (per-lcore). - */ -struct rte_ring_debug_stats { - uint64_t enq_success_bulk; /**< Successful enqueues number. */ - uint64_t enq_success_objs; /**< Objects successfully enqueued. */ - uint64_t enq_quota_bulk; /**< Successful enqueues above watermark. */ - uint64_t enq_quota_objs; /**< Objects enqueued above watermark. */ - uint64_t enq_fail_bulk; /**< Failed enqueues number. */ - uint64_t enq_fail_objs; /**< Objects that failed to be enqueued. */ - uint64_t deq_success_bulk; /**< Successful dequeues number. */ - uint64_t deq_success_objs; /**< Objects successfully dequeued. */ - uint64_t deq_fail_bulk; /**< Failed dequeues number. */ - uint64_t deq_fail_objs; /**< Objects that failed to be dequeued. */ -} __rte_cache_aligned; +#define RTE_RING_MZ_PREFIX "RG_" +/**< The maximum length of a ring name. */ +#define RTE_RING_NAMESIZE (RTE_MEMZONE_NAMESIZE - \ + sizeof(RTE_RING_MZ_PREFIX) + 1) + +struct rte_memzone; /* forward declaration, so as not to require memzone.h */ + +#if RTE_CACHE_LINE_SIZE < 128 +#define PROD_ALIGN (RTE_CACHE_LINE_SIZE * 2) +#define CONS_ALIGN (RTE_CACHE_LINE_SIZE * 2) +#else +#define PROD_ALIGN RTE_CACHE_LINE_SIZE +#define CONS_ALIGN RTE_CACHE_LINE_SIZE #endif -#define RTE_RING_NAMESIZE 32 /**< The maximum length of a ring name. */ -#define RTE_RING_MZ_PREFIX "RG_" +/* structure to hold a pair of head/tail values and other metadata */ +struct rte_ring_headtail { + volatile uint32_t head; /**< Prod/consumer head. */ + volatile uint32_t tail; /**< Prod/consumer tail. */ + uint32_t single; /**< True if single prod/cons */ +}; /** * An RTE ring structure. @@ -138,41 +142,28 @@ struct rte_ring_debug_stats { * a problem. */ struct rte_ring { - TAILQ_ENTRY(rte_ring) next; /**< Next in list. */ - - char name[RTE_RING_NAMESIZE]; /**< Name of the ring. */ - int flags; /**< Flags supplied at creation. */ + /* + * Note: this field kept the RTE_MEMZONE_NAMESIZE size due to ABI + * compatibility requirements, it could be changed to RTE_RING_NAMESIZE + * next time the ABI changes + */ + char name[RTE_MEMZONE_NAMESIZE]; /**< Name of the ring. */ + int flags; /**< Flags supplied at creation. */ + const struct rte_memzone *memzone; + /**< Memzone, if any, containing the rte_ring */ + uint32_t size; /**< Size of ring. */ + uint32_t mask; /**< Mask (size-1) of ring. */ + uint32_t watermark; /**< Max items before EDQUOT in producer. */ /** Ring producer status. */ - struct prod { - uint32_t watermark; /**< Maximum items before EDQUOT. */ - uint32_t sp_enqueue; /**< True, if single producer. */ - uint32_t size; /**< Size of ring. */ - uint32_t mask; /**< Mask (size-1) of ring. */ - volatile uint32_t head; /**< Producer head. */ - volatile uint32_t tail; /**< Producer tail. */ - } prod __rte_cache_aligned; + struct rte_ring_headtail prod __rte_aligned(PROD_ALIGN); /** Ring consumer status. */ - struct cons { - uint32_t sc_dequeue; /**< True, if single consumer. */ - uint32_t size; /**< Size of the ring. */ - uint32_t mask; /**< Mask (size-1) of ring. */ - volatile uint32_t head; /**< Consumer head. */ - volatile uint32_t tail; /**< Consumer tail. */ -#ifdef RTE_RING_SPLIT_PROD_CONS - } cons __rte_cache_aligned; -#else - } cons; -#endif - -#ifdef RTE_LIBRTE_RING_DEBUG - struct rte_ring_debug_stats stats[RTE_MAX_LCORE]; -#endif + struct rte_ring_headtail cons __rte_aligned(CONS_ALIGN); - void * ring[0] __rte_cache_aligned; /**< Memory space of ring starts here. - * not volatile so need to be careful - * about compiler re-ordering */ + void *ring[] __rte_cache_aligned; /**< Memory space of ring starts here. + * not volatile so need to be careful + * about compiler re-ordering */ }; #define RING_F_SP_ENQ 0x0001 /**< The default enqueue is "single-producer". */ @@ -180,25 +171,6 @@ struct rte_ring { #define RTE_RING_QUOT_EXCEED (1 << 31) /**< Quota exceed for burst ops */ #define RTE_RING_SZ_MASK (unsigned)(0x0fffffff) /**< Ring size mask */ -/** - * @internal When debug is enabled, store ring statistics. - * @param r - * A pointer to the ring. - * @param name - * The name of the statistics field to increment in the ring. - * @param n - * The number to add to the object-oriented statistics. - */ -#ifdef RTE_LIBRTE_RING_DEBUG -#define __RING_STAT_ADD(r, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - r->stats[__lcore_id].name##_objs += n; \ - r->stats[__lcore_id].name##_bulk += 1; \ - } while(0) -#else -#define __RING_STAT_ADD(r, name, n) do {} while(0) -#endif - /** * Calculate the memory size needed for a ring * @@ -286,7 +258,6 @@ int rte_ring_init(struct rte_ring *r, const char *name, unsigned count, * rte_errno set appropriately. Possible errno values include: * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure * - E_RTE_SECONDARY - function was called from a secondary process instance - * - E_RTE_NO_TAILQ - no tailq list could be got for the ring list * - EINVAL - count provided is not a power of 2 * - ENOSPC - the maximum number of memzones has already been allocated * - EEXIST - a memzone with the same name already exists @@ -294,6 +265,13 @@ int rte_ring_init(struct rte_ring *r, const char *name, unsigned count, */ struct rte_ring *rte_ring_create(const char *name, unsigned count, int socket_id, unsigned flags); +/** + * De-allocate all memory used by the ring. + * + * @param r + * Ring to free + */ +void rte_ring_free(struct rte_ring *r); /** * Change the high water mark. @@ -316,7 +294,7 @@ struct rte_ring *rte_ring_create(const char *name, unsigned count, int rte_ring_set_water_mark(struct rte_ring *r, unsigned count); /** - * Dump the status of the ring to the console. + * Dump the status of the ring to a file. * * @param f * A pointer to a file for output @@ -325,11 +303,11 @@ int rte_ring_set_water_mark(struct rte_ring *r, unsigned count); */ void rte_ring_dump(FILE *f, const struct rte_ring *r); -/* the actual enqueue of pointers on the ring. +/* the actual enqueue of pointers on the ring. * Placed here since identical code needed in both * single and multi producer enqueue functions */ #define ENQUEUE_PTRS() do { \ - const uint32_t size = r->prod.size; \ + const uint32_t size = r->size; \ uint32_t idx = prod_head & mask; \ if (likely(idx + n < size)) { \ for (i = 0; i < (n & ((~(unsigned)0x3))); i+=4, idx+=4) { \ @@ -351,12 +329,12 @@ void rte_ring_dump(FILE *f, const struct rte_ring *r); } \ } while(0) -/* the actual copy of pointers on the ring to obj_table. +/* the actual copy of pointers on the ring to obj_table. * Placed here since identical code needed in both * single and multi consumer dequeue functions */ #define DEQUEUE_PTRS() do { \ uint32_t idx = cons_head & mask; \ - const uint32_t size = r->cons.size; \ + const uint32_t size = r->size; \ if (likely(idx + n < size)) { \ for (i = 0; i < (n & (~(unsigned)0x3)); i+=4, idx+=4) {\ obj_table[i] = r->ring[idx]; \ @@ -410,10 +388,15 @@ __rte_ring_mp_do_enqueue(struct rte_ring *r, void * const *obj_table, uint32_t cons_tail, free_entries; const unsigned max = n; int success; - unsigned i; - uint32_t mask = r->prod.mask; + unsigned int i; + uint32_t mask = r->mask; int ret; + /* Avoid the unnecessary cmpset operation below, which is also + * potentially harmful when n equals 0. */ + if (n == 0) + return 0; + /* move prod.head atomically */ do { /* Reset n to the initial burst count */ @@ -429,17 +412,12 @@ __rte_ring_mp_do_enqueue(struct rte_ring *r, void * const *obj_table, /* check that we have enough room in ring */ if (unlikely(n > free_entries)) { - if (behavior == RTE_RING_QUEUE_FIXED) { - __RING_STAT_ADD(r, enq_fail, n); + if (behavior == RTE_RING_QUEUE_FIXED) return -ENOBUFS; - } else { /* No free entry available */ - if (unlikely(free_entries == 0)) { - __RING_STAT_ADD(r, enq_fail, n); + if (unlikely(free_entries == 0)) return 0; - } - n = free_entries; } } @@ -451,18 +429,14 @@ __rte_ring_mp_do_enqueue(struct rte_ring *r, void * const *obj_table, /* write entries in ring */ ENQUEUE_PTRS(); - rte_compiler_barrier(); + rte_smp_wmb(); /* if we exceed the watermark */ - if (unlikely(((mask + 1) - free_entries + n) > r->prod.watermark)) { + if (unlikely(((mask + 1) - free_entries + n) > r->watermark)) ret = (behavior == RTE_RING_QUEUE_FIXED) ? -EDQUOT : (int)(n | RTE_RING_QUOT_EXCEED); - __RING_STAT_ADD(r, enq_quota, n); - } - else { + else ret = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : n; - __RING_STAT_ADD(r, enq_success, n); - } /* * If there are other enqueues in progress that preceded us, @@ -503,8 +477,8 @@ __rte_ring_sp_do_enqueue(struct rte_ring *r, void * const *obj_table, { uint32_t prod_head, cons_tail; uint32_t prod_next, free_entries; - unsigned i; - uint32_t mask = r->prod.mask; + unsigned int i; + uint32_t mask = r->mask; int ret; prod_head = r->prod.head; @@ -517,17 +491,12 @@ __rte_ring_sp_do_enqueue(struct rte_ring *r, void * const *obj_table, /* check that we have enough room in ring */ if (unlikely(n > free_entries)) { - if (behavior == RTE_RING_QUEUE_FIXED) { - __RING_STAT_ADD(r, enq_fail, n); + if (behavior == RTE_RING_QUEUE_FIXED) return -ENOBUFS; - } else { /* No free entry available */ - if (unlikely(free_entries == 0)) { - __RING_STAT_ADD(r, enq_fail, n); + if (unlikely(free_entries == 0)) return 0; - } - n = free_entries; } } @@ -537,18 +506,14 @@ __rte_ring_sp_do_enqueue(struct rte_ring *r, void * const *obj_table, /* write entries in ring */ ENQUEUE_PTRS(); - rte_compiler_barrier(); + rte_smp_wmb(); /* if we exceed the watermark */ - if (unlikely(((mask + 1) - free_entries + n) > r->prod.watermark)) { + if (unlikely(((mask + 1) - free_entries + n) > r->watermark)) ret = (behavior == RTE_RING_QUEUE_FIXED) ? -EDQUOT : (int)(n | RTE_RING_QUOT_EXCEED); - __RING_STAT_ADD(r, enq_quota, n); - } - else { + else ret = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : n; - __RING_STAT_ADD(r, enq_success, n); - } r->prod.tail = prod_next; return ret; @@ -589,8 +554,13 @@ __rte_ring_mc_do_dequeue(struct rte_ring *r, void **obj_table, uint32_t cons_next, entries; const unsigned max = n; int success; - unsigned i; - uint32_t mask = r->prod.mask; + unsigned int i; + uint32_t mask = r->mask; + + /* Avoid the unnecessary cmpset operation below, which is also + * potentially harmful when n equals 0. */ + if (n == 0) + return 0; /* move cons.head atomically */ do { @@ -607,16 +577,11 @@ __rte_ring_mc_do_dequeue(struct rte_ring *r, void **obj_table, /* Set the actual entries for dequeue */ if (n > entries) { - if (behavior == RTE_RING_QUEUE_FIXED) { - __RING_STAT_ADD(r, deq_fail, n); + if (behavior == RTE_RING_QUEUE_FIXED) return -ENOENT; - } else { - if (unlikely(entries == 0)){ - __RING_STAT_ADD(r, deq_fail, n); + if (unlikely(entries == 0)) return 0; - } - n = entries; } } @@ -628,7 +593,7 @@ __rte_ring_mc_do_dequeue(struct rte_ring *r, void **obj_table, /* copy in table */ DEQUEUE_PTRS(); - rte_compiler_barrier(); + rte_smp_rmb(); /* * If there are other dequeues in progress that preceded us, @@ -637,7 +602,6 @@ __rte_ring_mc_do_dequeue(struct rte_ring *r, void **obj_table, while (unlikely(r->cons.tail != cons_head)) rte_pause(); - __RING_STAT_ADD(r, deq_success, n); r->cons.tail = cons_next; return behavior == RTE_RING_QUEUE_FIXED ? 0 : n; @@ -672,8 +636,8 @@ __rte_ring_sc_do_dequeue(struct rte_ring *r, void **obj_table, { uint32_t cons_head, prod_tail; uint32_t cons_next, entries; - unsigned i; - uint32_t mask = r->prod.mask; + unsigned int i; + uint32_t mask = r->mask; cons_head = r->cons.head; prod_tail = r->prod.tail; @@ -684,16 +648,11 @@ __rte_ring_sc_do_dequeue(struct rte_ring *r, void **obj_table, entries = prod_tail - cons_head; if (n > entries) { - if (behavior == RTE_RING_QUEUE_FIXED) { - __RING_STAT_ADD(r, deq_fail, n); + if (behavior == RTE_RING_QUEUE_FIXED) return -ENOENT; - } else { - if (unlikely(entries == 0)){ - __RING_STAT_ADD(r, deq_fail, n); + if (unlikely(entries == 0)) return 0; - } - n = entries; } } @@ -703,9 +662,8 @@ __rte_ring_sc_do_dequeue(struct rte_ring *r, void **obj_table, /* copy in table */ DEQUEUE_PTRS(); - rte_compiler_barrier(); + rte_smp_rmb(); - __RING_STAT_ADD(r, deq_success, n); r->cons.tail = cons_next; return behavior == RTE_RING_QUEUE_FIXED ? 0 : n; } @@ -780,7 +738,7 @@ static inline int __attribute__((always_inline)) rte_ring_enqueue_bulk(struct rte_ring *r, void * const *obj_table, unsigned n) { - if (r->prod.sp_enqueue) + if (r->prod.single) return rte_ring_sp_enqueue_bulk(r, obj_table, n); else return rte_ring_mp_enqueue_bulk(r, obj_table, n); @@ -847,7 +805,7 @@ rte_ring_sp_enqueue(struct rte_ring *r, void *obj) static inline int __attribute__((always_inline)) rte_ring_enqueue(struct rte_ring *r, void *obj) { - if (r->prod.sp_enqueue) + if (r->prod.single) return rte_ring_sp_enqueue(r, obj); else return rte_ring_mp_enqueue(r, obj); @@ -918,7 +876,7 @@ rte_ring_sc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned n) static inline int __attribute__((always_inline)) rte_ring_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned n) { - if (r->cons.sc_dequeue) + if (r->cons.single) return rte_ring_sc_dequeue_bulk(r, obj_table, n); else return rte_ring_mc_dequeue_bulk(r, obj_table, n); @@ -982,7 +940,7 @@ rte_ring_sc_dequeue(struct rte_ring *r, void **obj_p) static inline int __attribute__((always_inline)) rte_ring_dequeue(struct rte_ring *r, void **obj_p) { - if (r->cons.sc_dequeue) + if (r->cons.single) return rte_ring_sc_dequeue(r, obj_p); else return rte_ring_mc_dequeue(r, obj_p); @@ -1002,7 +960,7 @@ rte_ring_full(const struct rte_ring *r) { uint32_t prod_tail = r->prod.tail; uint32_t cons_tail = r->cons.tail; - return (((cons_tail - prod_tail - 1) & r->prod.mask) == 0); + return ((cons_tail - prod_tail - 1) & r->mask) == 0; } /** @@ -1035,7 +993,7 @@ rte_ring_count(const struct rte_ring *r) { uint32_t prod_tail = r->prod.tail; uint32_t cons_tail = r->cons.tail; - return ((prod_tail - cons_tail) & r->prod.mask); + return (prod_tail - cons_tail) & r->mask; } /** @@ -1051,7 +1009,21 @@ rte_ring_free_count(const struct rte_ring *r) { uint32_t prod_tail = r->prod.tail; uint32_t cons_tail = r->cons.tail; - return ((cons_tail - prod_tail - 1) & r->prod.mask); + return (cons_tail - prod_tail - 1) & r->mask; +} + +/** + * Return the size of the ring. + * + * @param r + * A pointer to the ring structure. + * @return + * The number of elements which can be stored in the ring. + */ +static inline unsigned int +rte_ring_get_size(const struct rte_ring *r) +{ + return r->size; } /** @@ -1089,7 +1061,7 @@ struct rte_ring *rte_ring_lookup(const char *name); * @return * - n: Actual number of objects enqueued. */ -static inline int __attribute__((always_inline)) +static inline unsigned __attribute__((always_inline)) rte_ring_mp_enqueue_burst(struct rte_ring *r, void * const *obj_table, unsigned n) { @@ -1108,7 +1080,7 @@ rte_ring_mp_enqueue_burst(struct rte_ring *r, void * const *obj_table, * @return * - n: Actual number of objects enqueued. */ -static inline int __attribute__((always_inline)) +static inline unsigned __attribute__((always_inline)) rte_ring_sp_enqueue_burst(struct rte_ring *r, void * const *obj_table, unsigned n) { @@ -1131,14 +1103,14 @@ rte_ring_sp_enqueue_burst(struct rte_ring *r, void * const *obj_table, * @return * - n: Actual number of objects enqueued. */ -static inline int __attribute__((always_inline)) +static inline unsigned __attribute__((always_inline)) rte_ring_enqueue_burst(struct rte_ring *r, void * const *obj_table, unsigned n) { - if (r->prod.sp_enqueue) - return rte_ring_sp_enqueue_burst(r, obj_table, n); + if (r->prod.single) + return rte_ring_sp_enqueue_burst(r, obj_table, n); else - return rte_ring_mp_enqueue_burst(r, obj_table, n); + return rte_ring_mp_enqueue_burst(r, obj_table, n); } /** @@ -1158,7 +1130,7 @@ rte_ring_enqueue_burst(struct rte_ring *r, void * const *obj_table, * @return * - n: Actual number of objects dequeued, 0 if ring is empty */ -static inline int __attribute__((always_inline)) +static inline unsigned __attribute__((always_inline)) rte_ring_mc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned n) { return __rte_ring_mc_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_VARIABLE); @@ -1178,7 +1150,7 @@ rte_ring_mc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned n) * @return * - n: Actual number of objects dequeued, 0 if ring is empty */ -static inline int __attribute__((always_inline)) +static inline unsigned __attribute__((always_inline)) rte_ring_sc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned n) { return __rte_ring_sc_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_VARIABLE); @@ -1198,12 +1170,12 @@ rte_ring_sc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned n) * @param n * The number of objects to dequeue from the ring to the obj_table. * @return - * - Number of objects dequeued, or a negative error code on error + * - Number of objects dequeued */ -static inline int __attribute__((always_inline)) +static inline unsigned __attribute__((always_inline)) rte_ring_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned n) { - if (r->cons.sc_dequeue) + if (r->cons.single) return rte_ring_sc_dequeue_burst(r, obj_table, n); else return rte_ring_mc_dequeue_burst(r, obj_table, n);