4 * Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 * Derived from FreeBSD's bufring.h
37 **************************************************************************
39 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
40 * All rights reserved.
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions are met:
45 * 1. Redistributions of source code must retain the above copyright notice,
46 * this list of conditions and the following disclaimer.
48 * 2. The name of Kip Macy nor the names of other
49 * contributors may be used to endorse or promote products derived from
50 * this software without specific prior written permission.
52 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
53 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
54 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
55 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
56 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
57 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
58 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
59 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
60 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
61 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
62 * POSSIBILITY OF SUCH DAMAGE.
64 ***************************************************************************/
66 #ifndef _RTE_RING_GENERIC_H_
67 #define _RTE_RING_GENERIC_H_
69 static __rte_always_inline void
70 update_tail(struct rte_ring_headtail *ht, uint32_t old_val, uint32_t new_val,
71 uint32_t single, uint32_t enqueue)
78 * If there are other enqueues/dequeues in progress that preceded us,
79 * we need to wait for them to complete
82 while (unlikely(ht->tail != old_val))
89 * @internal This function updates the producer head for enqueue
92 * A pointer to the ring structure
94 * Indicates whether multi-producer path is needed or not
96 * The number of elements we will want to enqueue, i.e. how far should the
99 * RTE_RING_QUEUE_FIXED: Enqueue a fixed number of items from a ring
100 * RTE_RING_QUEUE_VARIABLE: Enqueue as many items as possible from ring
102 * Returns head value as it was before the move, i.e. where enqueue starts
104 * Returns the current/new head value i.e. where enqueue finishes
105 * @param free_entries
106 * Returns the amount of free space in the ring BEFORE head was moved
108 * Actual number of objects enqueued.
109 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
111 static __rte_always_inline unsigned int
112 __rte_ring_move_prod_head(struct rte_ring *r, int is_sp,
113 unsigned int n, enum rte_ring_queue_behavior behavior,
114 uint32_t *old_head, uint32_t *new_head,
115 uint32_t *free_entries)
117 const uint32_t capacity = r->capacity;
118 unsigned int max = n;
122 /* Reset n to the initial burst count */
125 *old_head = r->prod.head;
127 /* add rmb barrier to avoid load/load reorder in weak
128 * memory model. It is noop on x86
132 const uint32_t cons_tail = r->cons.tail;
134 * The subtraction is done between two unsigned 32bits value
135 * (the result is always modulo 32 bits even if we have
136 * *old_head > cons_tail). So 'free_entries' is always between 0
137 * and capacity (which is < size).
139 *free_entries = (capacity + cons_tail - *old_head);
141 /* check that we have enough room in ring */
142 if (unlikely(n > *free_entries))
143 n = (behavior == RTE_RING_QUEUE_FIXED) ?
149 *new_head = *old_head + n;
151 r->prod.head = *new_head, success = 1;
153 success = rte_atomic32_cmpset(&r->prod.head,
154 *old_head, *new_head);
155 } while (unlikely(success == 0));
160 * @internal This function updates the consumer head for dequeue
163 * A pointer to the ring structure
165 * Indicates whether multi-consumer path is needed or not
167 * The number of elements we will want to enqueue, i.e. how far should the
170 * RTE_RING_QUEUE_FIXED: Dequeue a fixed number of items from a ring
171 * RTE_RING_QUEUE_VARIABLE: Dequeue as many items as possible from ring
173 * Returns head value as it was before the move, i.e. where dequeue starts
175 * Returns the current/new head value i.e. where dequeue finishes
177 * Returns the number of entries in the ring BEFORE head was moved
179 * - Actual number of objects dequeued.
180 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
182 static __rte_always_inline unsigned int
183 __rte_ring_move_cons_head(struct rte_ring *r, int is_sc,
184 unsigned int n, enum rte_ring_queue_behavior behavior,
185 uint32_t *old_head, uint32_t *new_head,
188 unsigned int max = n;
191 /* move cons.head atomically */
193 /* Restore n as it may change every loop */
196 *old_head = r->cons.head;
198 /* add rmb barrier to avoid load/load reorder in weak
199 * memory model. It is noop on x86
203 const uint32_t prod_tail = r->prod.tail;
204 /* The subtraction is done between two unsigned 32bits value
205 * (the result is always modulo 32 bits even if we have
206 * cons_head > prod_tail). So 'entries' is always between 0
209 *entries = (prod_tail - *old_head);
211 /* Set the actual entries for dequeue */
213 n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries;
215 if (unlikely(n == 0))
218 *new_head = *old_head + n;
220 r->cons.head = *new_head, success = 1;
222 success = rte_atomic32_cmpset(&r->cons.head, *old_head,
224 } while (unlikely(success == 0));
228 #endif /* _RTE_RING_GENERIC_H_ */