ring: make struct and macros type agnostic

[dpdk.git] / lib / librte_ring / rte_ring.h

diff --git a/lib/librte_ring/rte_ring.h b/lib/librte_ring/rte_ring.h
index fa66638..6642e18 100644 (file)
--- a/lib/librte_ring/rte_ring.h
+++ b/lib/librte_ring/rte_ring.h
@@ -159,11 +159,7 @@ struct rte_ring {
 
        /** Ring consumer status. */
        struct rte_ring_headtail cons __rte_aligned(CONS_ALIGN);
-
-       void *ring[] __rte_cache_aligned;   /**< Memory space of ring starts here.
-                                            * not volatile so need to be careful
-                                            * about compiler re-ordering */
-};
+} __rte_cache_aligned;
 
 #define RING_F_SP_ENQ 0x0001 /**< The default enqueue is "single-producer". */
 #define RING_F_SC_DEQ 0x0002 /**< The default dequeue is "single-consumer". */
@@ -290,57 +286,80 @@ void rte_ring_dump(FILE *f, const struct rte_ring *r);
 /* the actual enqueue of pointers on the ring.
  * Placed here since identical code needed in both
  * single and multi producer enqueue functions */
-#define ENQUEUE_PTRS() do { \
+#define ENQUEUE_PTRS(r, ring_start, prod_head, obj_table, n, obj_type) do { \
        unsigned int i; \
-       const uint32_t size = r->size; \
-       uint32_t idx = prod_head & r->mask; \
+       const uint32_t size = (r)->size; \
+       uint32_t idx = prod_head & (r)->mask; \
+       obj_type *ring = (void *)ring_start; \
        if (likely(idx + n < size)) { \
                for (i = 0; i < (n & ((~(unsigned)0x3))); i+=4, idx+=4) { \
-                       r->ring[idx] = obj_table[i]; \
-                       r->ring[idx+1] = obj_table[i+1]; \
-                       r->ring[idx+2] = obj_table[i+2]; \
-                       r->ring[idx+3] = obj_table[i+3]; \
+                       ring[idx] = obj_table[i]; \
+                       ring[idx+1] = obj_table[i+1]; \
+                       ring[idx+2] = obj_table[i+2]; \
+                       ring[idx+3] = obj_table[i+3]; \
                } \
                switch (n & 0x3) { \
-                       case 3: r->ring[idx++] = obj_table[i++]; \
-                       case 2: r->ring[idx++] = obj_table[i++]; \
-                       case 1: r->ring[idx++] = obj_table[i++]; \
+               case 3: \
+                       ring[idx++] = obj_table[i++]; /* fallthrough */ \
+               case 2: \
+                       ring[idx++] = obj_table[i++]; /* fallthrough */ \
+               case 1: \
+                       ring[idx++] = obj_table[i++]; \
                } \
        } else { \
                for (i = 0; idx < size; i++, idx++)\
-                       r->ring[idx] = obj_table[i]; \
+                       ring[idx] = obj_table[i]; \
                for (idx = 0; i < n; i++, idx++) \
-                       r->ring[idx] = obj_table[i]; \
+                       ring[idx] = obj_table[i]; \
        } \
-} while(0)
+} while (0)
 
 /* the actual copy of pointers on the ring to obj_table.
  * Placed here since identical code needed in both
  * single and multi consumer dequeue functions */
-#define DEQUEUE_PTRS() do { \
+#define DEQUEUE_PTRS(r, ring_start, cons_head, obj_table, n, obj_type) do { \
        unsigned int i; \
-       uint32_t idx = cons_head & r->mask; \
-       const uint32_t size = r->size; \
+       uint32_t idx = cons_head & (r)->mask; \
+       const uint32_t size = (r)->size; \
+       obj_type *ring = (void *)ring_start; \
        if (likely(idx + n < size)) { \
                for (i = 0; i < (n & (~(unsigned)0x3)); i+=4, idx+=4) {\
-                       obj_table[i] = r->ring[idx]; \
-                       obj_table[i+1] = r->ring[idx+1]; \
-                       obj_table[i+2] = r->ring[idx+2]; \
-                       obj_table[i+3] = r->ring[idx+3]; \
+                       obj_table[i] = ring[idx]; \
+                       obj_table[i+1] = ring[idx+1]; \
+                       obj_table[i+2] = ring[idx+2]; \
+                       obj_table[i+3] = ring[idx+3]; \
                } \
                switch (n & 0x3) { \
-                       case 3: obj_table[i++] = r->ring[idx++]; \
-                       case 2: obj_table[i++] = r->ring[idx++]; \
-                       case 1: obj_table[i++] = r->ring[idx++]; \
+               case 3: \
+                       obj_table[i++] = ring[idx++]; /* fallthrough */ \
+               case 2: \
+                       obj_table[i++] = ring[idx++]; /* fallthrough */ \
+               case 1: \
+                       obj_table[i++] = ring[idx++]; \
                } \
        } else { \
                for (i = 0; idx < size; i++, idx++) \
-                       obj_table[i] = r->ring[idx]; \
+                       obj_table[i] = ring[idx]; \
                for (idx = 0; i < n; i++, idx++) \
-                       obj_table[i] = r->ring[idx]; \
+                       obj_table[i] = ring[idx]; \
        } \
 } while (0)
 
+static inline __attribute__((always_inline)) void
+update_tail(struct rte_ring_headtail *ht, uint32_t old_val, uint32_t new_val,
+               uint32_t single)
+{
+       /*
+        * If there are other enqueues/dequeues in progress that preceded us,
+        * we need to wait for them to complete
+        */
+       if (!single)
+               while (unlikely(ht->tail != old_val))
+                       rte_pause();
+
+       ht->tail = new_val;
+}
+
 /**
  * @internal This function updates the producer head for enqueue
  *
@@ -437,18 +456,10 @@ __rte_ring_do_enqueue(struct rte_ring *r, void * const *obj_table,
        if (n == 0)
                goto end;
 
-       ENQUEUE_PTRS();
+       ENQUEUE_PTRS(r, &r[1], prod_head, obj_table, n, void *);
        rte_smp_wmb();
 
-       /*
-        * If there are other enqueues in progress that preceded us,
-        * we need to wait for them to complete
-        */
-       while (unlikely(r->prod.tail != prod_head))
-               rte_pause();
-
-       r->prod.tail = prod_next;
-
+       update_tail(&r->prod, prod_head, prod_next, is_sp);
 end:
        if (free_space != NULL)
                *free_space = free_entries - n;
@@ -550,17 +561,10 @@ __rte_ring_do_dequeue(struct rte_ring *r, void **obj_table,
        if (n == 0)
                goto end;
 
-       DEQUEUE_PTRS();
+       DEQUEUE_PTRS(r, &r[1], cons_head, obj_table, n, void *);
        rte_smp_rmb();
 
-       /*
-        * If there are other enqueues in progress that preceded us,
-        * we need to wait for them to complete
-        */
-       while (unlikely(r->cons.tail != cons_head))
-               rte_pause();
-
-       r->cons.tail = cons_next;
+       update_tail(&r->cons, cons_head, cons_next, is_sc);
 
 end:
        if (available != NULL)