1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2017,2018 HXT-semitech Corporation.
4 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
6 * Derived from FreeBSD's bufring.h
7 * Used as BSD-3 Licensed with permission from Kip Macy.
10 #ifndef _RTE_RING_C11_MEM_H_
11 #define _RTE_RING_C11_MEM_H_
13 static __rte_always_inline void
14 update_tail(struct rte_ring_headtail *ht, uint32_t old_val, uint32_t new_val,
15 uint32_t single, uint32_t enqueue)
17 RTE_SET_USED(enqueue);
20 * If there are other enqueues/dequeues in progress that preceded us,
21 * we need to wait for them to complete
24 while (unlikely(ht->tail != old_val))
27 __atomic_store_n(&ht->tail, new_val, __ATOMIC_RELEASE);
31 * @internal This function updates the producer head for enqueue
34 * A pointer to the ring structure
36 * Indicates whether multi-producer path is needed or not
38 * The number of elements we will want to enqueue, i.e. how far should the
41 * RTE_RING_QUEUE_FIXED: Enqueue a fixed number of items from a ring
42 * RTE_RING_QUEUE_VARIABLE: Enqueue as many items as possible from ring
44 * Returns head value as it was before the move, i.e. where enqueue starts
46 * Returns the current/new head value i.e. where enqueue finishes
48 * Returns the amount of free space in the ring BEFORE head was moved
50 * Actual number of objects enqueued.
51 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
53 static __rte_always_inline unsigned int
54 __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp,
55 unsigned int n, enum rte_ring_queue_behavior behavior,
56 uint32_t *old_head, uint32_t *new_head,
57 uint32_t *free_entries)
59 const uint32_t capacity = r->capacity;
64 *old_head = __atomic_load_n(&r->prod.head, __ATOMIC_ACQUIRE);
66 /* Reset n to the initial burst count */
69 /* load-acquire synchronize with store-release of ht->tail
72 cons_tail = __atomic_load_n(&r->cons.tail,
75 /* The subtraction is done between two unsigned 32bits value
76 * (the result is always modulo 32 bits even if we have
77 * *old_head > cons_tail). So 'free_entries' is always between 0
78 * and capacity (which is < size).
80 *free_entries = (capacity + cons_tail - *old_head);
82 /* check that we have enough room in ring */
83 if (unlikely(n > *free_entries))
84 n = (behavior == RTE_RING_QUEUE_FIXED) ?
90 *new_head = *old_head + n;
92 r->prod.head = *new_head, success = 1;
94 /* on failure, *old_head is updated */
95 success = __atomic_compare_exchange_n(&r->prod.head,
99 } while (unlikely(success == 0));
104 * @internal This function updates the consumer head for dequeue
107 * A pointer to the ring structure
109 * Indicates whether multi-consumer path is needed or not
111 * The number of elements we will want to enqueue, i.e. how far should the
114 * RTE_RING_QUEUE_FIXED: Dequeue a fixed number of items from a ring
115 * RTE_RING_QUEUE_VARIABLE: Dequeue as many items as possible from ring
117 * Returns head value as it was before the move, i.e. where dequeue starts
119 * Returns the current/new head value i.e. where dequeue finishes
121 * Returns the number of entries in the ring BEFORE head was moved
123 * - Actual number of objects dequeued.
124 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
126 static __rte_always_inline unsigned int
127 __rte_ring_move_cons_head(struct rte_ring *r, int is_sc,
128 unsigned int n, enum rte_ring_queue_behavior behavior,
129 uint32_t *old_head, uint32_t *new_head,
132 unsigned int max = n;
136 /* move cons.head atomically */
137 *old_head = __atomic_load_n(&r->cons.head, __ATOMIC_ACQUIRE);
139 /* Restore n as it may change every loop */
142 /* this load-acquire synchronize with store-release of ht->tail
145 prod_tail = __atomic_load_n(&r->prod.tail,
148 /* The subtraction is done between two unsigned 32bits value
149 * (the result is always modulo 32 bits even if we have
150 * cons_head > prod_tail). So 'entries' is always between 0
153 *entries = (prod_tail - *old_head);
155 /* Set the actual entries for dequeue */
157 n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries;
159 if (unlikely(n == 0))
162 *new_head = *old_head + n;
164 r->cons.head = *new_head, success = 1;
166 /* on failure, *old_head will be updated */
167 success = __atomic_compare_exchange_n(&r->cons.head,
171 } while (unlikely(success == 0));
175 #endif /* _RTE_RING_C11_MEM_H_ */