1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2010-2017 Intel Corporation
4 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
6 * Derived from FreeBSD's bufring.h
7 * Used as BSD-3 Licensed with permission from Kip Macy.
10 #ifndef _RTE_RING_GENERIC_PVT_H_
11 #define _RTE_RING_GENERIC_PVT_H_
13 static __rte_always_inline void
14 __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,
15 uint32_t new_val, uint32_t single, uint32_t enqueue)
22 * If there are other enqueues/dequeues in progress that preceded us,
23 * we need to wait for them to complete
26 rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
32 * @internal This function updates the producer head for enqueue
35 * A pointer to the ring structure
37 * Indicates whether multi-producer path is needed or not
39 * The number of elements we will want to enqueue, i.e. how far should the
42 * RTE_RING_QUEUE_FIXED: Enqueue a fixed number of items from a ring
43 * RTE_RING_QUEUE_VARIABLE: Enqueue as many items as possible from ring
45 * Returns head value as it was before the move, i.e. where enqueue starts
47 * Returns the current/new head value i.e. where enqueue finishes
49 * Returns the amount of free space in the ring BEFORE head was moved
51 * Actual number of objects enqueued.
52 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
54 static __rte_always_inline unsigned int
55 __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp,
56 unsigned int n, enum rte_ring_queue_behavior behavior,
57 uint32_t *old_head, uint32_t *new_head,
58 uint32_t *free_entries)
60 const uint32_t capacity = r->capacity;
65 /* Reset n to the initial burst count */
68 *old_head = r->prod.head;
70 /* add rmb barrier to avoid load/load reorder in weak
71 * memory model. It is noop on x86
76 * The subtraction is done between two unsigned 32bits value
77 * (the result is always modulo 32 bits even if we have
78 * *old_head > cons_tail). So 'free_entries' is always between 0
79 * and capacity (which is < size).
81 *free_entries = (capacity + r->cons.tail - *old_head);
83 /* check that we have enough room in ring */
84 if (unlikely(n > *free_entries))
85 n = (behavior == RTE_RING_QUEUE_FIXED) ?
91 *new_head = *old_head + n;
93 r->prod.head = *new_head, success = 1;
95 success = rte_atomic32_cmpset(&r->prod.head,
96 *old_head, *new_head);
97 } while (unlikely(success == 0));
102 * @internal This function updates the consumer head for dequeue
105 * A pointer to the ring structure
107 * Indicates whether multi-consumer path is needed or not
109 * The number of elements we will want to enqueue, i.e. how far should the
112 * RTE_RING_QUEUE_FIXED: Dequeue a fixed number of items from a ring
113 * RTE_RING_QUEUE_VARIABLE: Dequeue as many items as possible from ring
115 * Returns head value as it was before the move, i.e. where dequeue starts
117 * Returns the current/new head value i.e. where dequeue finishes
119 * Returns the number of entries in the ring BEFORE head was moved
121 * - Actual number of objects dequeued.
122 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
124 static __rte_always_inline unsigned int
125 __rte_ring_move_cons_head(struct rte_ring *r, unsigned int is_sc,
126 unsigned int n, enum rte_ring_queue_behavior behavior,
127 uint32_t *old_head, uint32_t *new_head,
130 unsigned int max = n;
133 /* move cons.head atomically */
135 /* Restore n as it may change every loop */
138 *old_head = r->cons.head;
140 /* add rmb barrier to avoid load/load reorder in weak
141 * memory model. It is noop on x86
145 /* The subtraction is done between two unsigned 32bits value
146 * (the result is always modulo 32 bits even if we have
147 * cons_head > prod_tail). So 'entries' is always between 0
150 *entries = (r->prod.tail - *old_head);
152 /* Set the actual entries for dequeue */
154 n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries;
156 if (unlikely(n == 0))
159 *new_head = *old_head + n;
161 r->cons.head = *new_head;
165 success = rte_atomic32_cmpset(&r->cons.head, *old_head,
168 } while (unlikely(success == 0));
172 #endif /* _RTE_RING_GENERIC_PVT_H_ */