4 * Copyright(c) 2017 HXT-semitech Corporation.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
16 * * Neither the name of HXT-semitech Corporation nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 * Derived from FreeBSD's bufring.h
36 **************************************************************************
38 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
39 * All rights reserved.
41 * Redistribution and use in source and binary forms, with or without
42 * modification, are permitted provided that the following conditions are met:
44 * 1. Redistributions of source code must retain the above copyright notice,
45 * this list of conditions and the following disclaimer.
47 * 2. The name of Kip Macy nor the names of other
48 * contributors may be used to endorse or promote products derived from
49 * this software without specific prior written permission.
51 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
52 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
54 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
55 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
56 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
57 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
58 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
59 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
60 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
61 * POSSIBILITY OF SUCH DAMAGE.
63 ***************************************************************************/
65 #ifndef _RTE_RING_C11_MEM_H_
66 #define _RTE_RING_C11_MEM_H_
68 static __rte_always_inline void
69 update_tail(struct rte_ring_headtail *ht, uint32_t old_val, uint32_t new_val,
70 uint32_t single, uint32_t enqueue)
72 RTE_SET_USED(enqueue);
75 * If there are other enqueues/dequeues in progress that preceded us,
76 * we need to wait for them to complete
79 while (unlikely(ht->tail != old_val))
82 __atomic_store_n(&ht->tail, new_val, __ATOMIC_RELEASE);
86 * @internal This function updates the producer head for enqueue
89 * A pointer to the ring structure
91 * Indicates whether multi-producer path is needed or not
93 * The number of elements we will want to enqueue, i.e. how far should the
96 * RTE_RING_QUEUE_FIXED: Enqueue a fixed number of items from a ring
97 * RTE_RING_QUEUE_VARIABLE: Enqueue as many items as possible from ring
99 * Returns head value as it was before the move, i.e. where enqueue starts
101 * Returns the current/new head value i.e. where enqueue finishes
102 * @param free_entries
103 * Returns the amount of free space in the ring BEFORE head was moved
105 * Actual number of objects enqueued.
106 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
108 static __rte_always_inline unsigned int
109 __rte_ring_move_prod_head(struct rte_ring *r, int is_sp,
110 unsigned int n, enum rte_ring_queue_behavior behavior,
111 uint32_t *old_head, uint32_t *new_head,
112 uint32_t *free_entries)
114 const uint32_t capacity = r->capacity;
115 unsigned int max = n;
119 /* Reset n to the initial burst count */
122 *old_head = __atomic_load_n(&r->prod.head,
124 const uint32_t cons_tail = r->cons.tail;
126 * The subtraction is done between two unsigned 32bits value
127 * (the result is always modulo 32 bits even if we have
128 * *old_head > cons_tail). So 'free_entries' is always between 0
129 * and capacity (which is < size).
131 *free_entries = (capacity + cons_tail - *old_head);
133 /* check that we have enough room in ring */
134 if (unlikely(n > *free_entries))
135 n = (behavior == RTE_RING_QUEUE_FIXED) ?
141 *new_head = *old_head + n;
143 r->prod.head = *new_head, success = 1;
145 success = __atomic_compare_exchange_n(&r->prod.head,
149 } while (unlikely(success == 0));
154 * @internal This function updates the consumer head for dequeue
157 * A pointer to the ring structure
159 * Indicates whether multi-consumer path is needed or not
161 * The number of elements we will want to enqueue, i.e. how far should the
164 * RTE_RING_QUEUE_FIXED: Dequeue a fixed number of items from a ring
165 * RTE_RING_QUEUE_VARIABLE: Dequeue as many items as possible from ring
167 * Returns head value as it was before the move, i.e. where dequeue starts
169 * Returns the current/new head value i.e. where dequeue finishes
171 * Returns the number of entries in the ring BEFORE head was moved
173 * - Actual number of objects dequeued.
174 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
176 static __rte_always_inline unsigned int
177 __rte_ring_move_cons_head(struct rte_ring *r, int is_sc,
178 unsigned int n, enum rte_ring_queue_behavior behavior,
179 uint32_t *old_head, uint32_t *new_head,
182 unsigned int max = n;
185 /* move cons.head atomically */
187 /* Restore n as it may change every loop */
189 *old_head = __atomic_load_n(&r->cons.head,
191 const uint32_t prod_tail = r->prod.tail;
192 /* The subtraction is done between two unsigned 32bits value
193 * (the result is always modulo 32 bits even if we have
194 * cons_head > prod_tail). So 'entries' is always between 0
197 *entries = (prod_tail - *old_head);
199 /* Set the actual entries for dequeue */
201 n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries;
203 if (unlikely(n == 0))
206 *new_head = *old_head + n;
208 r->cons.head = *new_head, success = 1;
210 success = __atomic_compare_exchange_n(&r->cons.head,
214 } while (unlikely(success == 0));
218 #endif /* _RTE_RING_C11_MEM_H_ */