1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2017,2018 HXT-semitech Corporation.
4 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
6 * Derived from FreeBSD's bufring.h
7 * Used as BSD-3 Licensed with permission from Kip Macy.
10 #ifndef _RTE_RING_C11_MEM_H_
11 #define _RTE_RING_C11_MEM_H_
13 static __rte_always_inline void
14 update_tail(struct rte_ring_headtail *ht, uint32_t old_val, uint32_t new_val,
15 uint32_t single, uint32_t enqueue)
17 RTE_SET_USED(enqueue);
20 * If there are other enqueues/dequeues in progress that preceded us,
21 * we need to wait for them to complete
24 while (unlikely(ht->tail != old_val))
27 __atomic_store_n(&ht->tail, new_val, __ATOMIC_RELEASE);
31 * @internal This function updates the producer head for enqueue
34 * A pointer to the ring structure
36 * Indicates whether multi-producer path is needed or not
38 * The number of elements we will want to enqueue, i.e. how far should the
41 * RTE_RING_QUEUE_FIXED: Enqueue a fixed number of items from a ring
42 * RTE_RING_QUEUE_VARIABLE: Enqueue as many items as possible from ring
44 * Returns head value as it was before the move, i.e. where enqueue starts
46 * Returns the current/new head value i.e. where enqueue finishes
48 * Returns the amount of free space in the ring BEFORE head was moved
50 * Actual number of objects enqueued.
51 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
53 static __rte_always_inline unsigned int
54 __rte_ring_move_prod_head(struct rte_ring *r, int is_sp,
55 unsigned int n, enum rte_ring_queue_behavior behavior,
56 uint32_t *old_head, uint32_t *new_head,
57 uint32_t *free_entries)
59 const uint32_t capacity = r->capacity;
64 /* Reset n to the initial burst count */
67 *old_head = __atomic_load_n(&r->prod.head,
69 const uint32_t cons_tail = r->cons.tail;
71 * The subtraction is done between two unsigned 32bits value
72 * (the result is always modulo 32 bits even if we have
73 * *old_head > cons_tail). So 'free_entries' is always between 0
74 * and capacity (which is < size).
76 *free_entries = (capacity + cons_tail - *old_head);
78 /* check that we have enough room in ring */
79 if (unlikely(n > *free_entries))
80 n = (behavior == RTE_RING_QUEUE_FIXED) ?
86 *new_head = *old_head + n;
88 r->prod.head = *new_head, success = 1;
90 success = __atomic_compare_exchange_n(&r->prod.head,
94 } while (unlikely(success == 0));
99 * @internal This function updates the consumer head for dequeue
102 * A pointer to the ring structure
104 * Indicates whether multi-consumer path is needed or not
106 * The number of elements we will want to enqueue, i.e. how far should the
109 * RTE_RING_QUEUE_FIXED: Dequeue a fixed number of items from a ring
110 * RTE_RING_QUEUE_VARIABLE: Dequeue as many items as possible from ring
112 * Returns head value as it was before the move, i.e. where dequeue starts
114 * Returns the current/new head value i.e. where dequeue finishes
116 * Returns the number of entries in the ring BEFORE head was moved
118 * - Actual number of objects dequeued.
119 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
121 static __rte_always_inline unsigned int
122 __rte_ring_move_cons_head(struct rte_ring *r, int is_sc,
123 unsigned int n, enum rte_ring_queue_behavior behavior,
124 uint32_t *old_head, uint32_t *new_head,
127 unsigned int max = n;
130 /* move cons.head atomically */
132 /* Restore n as it may change every loop */
134 *old_head = __atomic_load_n(&r->cons.head,
136 const uint32_t prod_tail = r->prod.tail;
137 /* The subtraction is done between two unsigned 32bits value
138 * (the result is always modulo 32 bits even if we have
139 * cons_head > prod_tail). So 'entries' is always between 0
142 *entries = (prod_tail - *old_head);
144 /* Set the actual entries for dequeue */
146 n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries;
148 if (unlikely(n == 0))
151 *new_head = *old_head + n;
153 r->cons.head = *new_head, success = 1;
155 success = __atomic_compare_exchange_n(&r->cons.head,
159 } while (unlikely(success == 0));
163 #endif /* _RTE_RING_C11_MEM_H_ */