1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2010-2020 Intel Corporation
4 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
6 * Derived from FreeBSD's bufring.h
7 * Used as BSD-3 Licensed with permission from Kip Macy.
10 #ifndef _RTE_RING_HTS_H_
11 #define _RTE_RING_HTS_H_
14 * @file rte_ring_hts.h
15 * @b EXPERIMENTAL: this API may change without prior notice
16 * It is not recommended to include this file directly.
17 * Please include <rte_ring.h> instead.
19 * Contains functions for serialized, aka Head-Tail Sync (HTS) ring mode.
20 * In that mode enqueue/dequeue operation is fully serialized:
21 * at any given moment only one enqueue/dequeue operation can proceed.
22 * This is achieved by allowing a thread to proceed with changing head.value
23 * only when head.value == tail.value.
24 * Both head and tail values are updated atomically (as one 64-bit value).
25 * To achieve that 64-bit CAS is used by head update routine.
32 #include <rte_ring_hts_c11_mem.h>
35 * @internal Enqueue several objects on the HTS ring.
38 * A pointer to the ring structure.
40 * A pointer to a table of objects.
42 * The size of ring element, in bytes. It must be a multiple of 4.
43 * This must be the same value used while creating the ring. Otherwise
44 * the results are undefined.
46 * The number of objects to add in the ring from the obj_table.
48 * RTE_RING_QUEUE_FIXED: Enqueue a fixed number of items from a ring
49 * RTE_RING_QUEUE_VARIABLE: Enqueue as many items as possible from ring
51 * returns the amount of space after the enqueue operation has finished
53 * Actual number of objects enqueued.
54 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
56 static __rte_always_inline unsigned int
57 __rte_ring_do_hts_enqueue_elem(struct rte_ring *r, const void *obj_table,
58 uint32_t esize, uint32_t n, enum rte_ring_queue_behavior behavior,
63 n = __rte_ring_hts_move_prod_head(r, n, behavior, &head, &free);
66 __rte_ring_enqueue_elems(r, head, obj_table, esize, n);
67 __rte_ring_hts_update_tail(&r->hts_prod, head, n, 1);
70 if (free_space != NULL)
71 *free_space = free - n;
76 * @internal Dequeue several objects from the HTS ring.
79 * A pointer to the ring structure.
81 * A pointer to a table of objects.
83 * The size of ring element, in bytes. It must be a multiple of 4.
84 * This must be the same value used while creating the ring. Otherwise
85 * the results are undefined.
87 * The number of objects to pull from the ring.
89 * RTE_RING_QUEUE_FIXED: Dequeue a fixed number of items from a ring
90 * RTE_RING_QUEUE_VARIABLE: Dequeue as many items as possible from ring
92 * returns the number of remaining ring entries after the dequeue has finished
94 * - Actual number of objects dequeued.
95 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
97 static __rte_always_inline unsigned int
98 __rte_ring_do_hts_dequeue_elem(struct rte_ring *r, void *obj_table,
99 uint32_t esize, uint32_t n, enum rte_ring_queue_behavior behavior,
102 uint32_t entries, head;
104 n = __rte_ring_hts_move_cons_head(r, n, behavior, &head, &entries);
107 __rte_ring_dequeue_elems(r, head, obj_table, esize, n);
108 __rte_ring_hts_update_tail(&r->hts_cons, head, n, 0);
111 if (available != NULL)
112 *available = entries - n;
117 * Enqueue several objects on the HTS ring (multi-producers safe).
120 * A pointer to the ring structure.
122 * A pointer to a table of objects.
124 * The size of ring element, in bytes. It must be a multiple of 4.
125 * This must be the same value used while creating the ring. Otherwise
126 * the results are undefined.
128 * The number of objects to add in the ring from the obj_table.
130 * if non-NULL, returns the amount of space in the ring after the
131 * enqueue operation has finished.
133 * The number of objects enqueued, either 0 or n
136 static __rte_always_inline unsigned int
137 rte_ring_mp_hts_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table,
138 unsigned int esize, unsigned int n, unsigned int *free_space)
140 return __rte_ring_do_hts_enqueue_elem(r, obj_table, esize, n,
141 RTE_RING_QUEUE_FIXED, free_space);
145 * Dequeue several objects from an HTS ring (multi-consumers safe).
148 * A pointer to the ring structure.
150 * A pointer to a table of objects that will be filled.
152 * The size of ring element, in bytes. It must be a multiple of 4.
153 * This must be the same value used while creating the ring. Otherwise
154 * the results are undefined.
156 * The number of objects to dequeue from the ring to the obj_table.
158 * If non-NULL, returns the number of remaining ring entries after the
159 * dequeue has finished.
161 * The number of objects dequeued, either 0 or n
164 static __rte_always_inline unsigned int
165 rte_ring_mc_hts_dequeue_bulk_elem(struct rte_ring *r, void *obj_table,
166 unsigned int esize, unsigned int n, unsigned int *available)
168 return __rte_ring_do_hts_dequeue_elem(r, obj_table, esize, n,
169 RTE_RING_QUEUE_FIXED, available);
173 * Enqueue several objects on the HTS ring (multi-producers safe).
176 * A pointer to the ring structure.
178 * A pointer to a table of objects.
180 * The size of ring element, in bytes. It must be a multiple of 4.
181 * This must be the same value used while creating the ring. Otherwise
182 * the results are undefined.
184 * The number of objects to add in the ring from the obj_table.
186 * if non-NULL, returns the amount of space in the ring after the
187 * enqueue operation has finished.
189 * - n: Actual number of objects enqueued.
192 static __rte_always_inline unsigned int
193 rte_ring_mp_hts_enqueue_burst_elem(struct rte_ring *r, const void *obj_table,
194 unsigned int esize, unsigned int n, unsigned int *free_space)
196 return __rte_ring_do_hts_enqueue_elem(r, obj_table, esize, n,
197 RTE_RING_QUEUE_VARIABLE, free_space);
201 * Dequeue several objects from an HTS ring (multi-consumers safe).
202 * When the requested objects are more than the available objects,
203 * only dequeue the actual number of objects.
206 * A pointer to the ring structure.
208 * A pointer to a table of objects that will be filled.
210 * The size of ring element, in bytes. It must be a multiple of 4.
211 * This must be the same value used while creating the ring. Otherwise
212 * the results are undefined.
214 * The number of objects to dequeue from the ring to the obj_table.
216 * If non-NULL, returns the number of remaining ring entries after the
217 * dequeue has finished.
219 * - n: Actual number of objects dequeued, 0 if ring is empty
222 static __rte_always_inline unsigned int
223 rte_ring_mc_hts_dequeue_burst_elem(struct rte_ring *r, void *obj_table,
224 unsigned int esize, unsigned int n, unsigned int *available)
226 return __rte_ring_do_hts_dequeue_elem(r, obj_table, esize, n,
227 RTE_RING_QUEUE_VARIABLE, available);
231 * Enqueue several objects on the HTS ring (multi-producers safe).
234 * A pointer to the ring structure.
236 * A pointer to a table of void * pointers (objects).
238 * The number of objects to add in the ring from the obj_table.
240 * if non-NULL, returns the amount of space in the ring after the
241 * enqueue operation has finished.
243 * The number of objects enqueued, either 0 or n
246 static __rte_always_inline unsigned int
247 rte_ring_mp_hts_enqueue_bulk(struct rte_ring *r, void * const *obj_table,
248 unsigned int n, unsigned int *free_space)
250 return rte_ring_mp_hts_enqueue_bulk_elem(r, obj_table,
251 sizeof(uintptr_t), n, free_space);
255 * Dequeue several objects from an HTS ring (multi-consumers safe).
258 * A pointer to the ring structure.
260 * A pointer to a table of void * pointers (objects) that will be filled.
262 * The number of objects to dequeue from the ring to the obj_table.
264 * If non-NULL, returns the number of remaining ring entries after the
265 * dequeue has finished.
267 * The number of objects dequeued, either 0 or n
270 static __rte_always_inline unsigned int
271 rte_ring_mc_hts_dequeue_bulk(struct rte_ring *r, void **obj_table,
272 unsigned int n, unsigned int *available)
274 return rte_ring_mc_hts_dequeue_bulk_elem(r, obj_table,
275 sizeof(uintptr_t), n, available);
279 * Enqueue several objects on the HTS ring (multi-producers safe).
282 * A pointer to the ring structure.
284 * A pointer to a table of void * pointers (objects).
286 * The number of objects to add in the ring from the obj_table.
288 * if non-NULL, returns the amount of space in the ring after the
289 * enqueue operation has finished.
291 * - n: Actual number of objects enqueued.
294 static __rte_always_inline unsigned int
295 rte_ring_mp_hts_enqueue_burst(struct rte_ring *r, void * const *obj_table,
296 unsigned int n, unsigned int *free_space)
298 return rte_ring_mp_hts_enqueue_burst_elem(r, obj_table,
299 sizeof(uintptr_t), n, free_space);
303 * Dequeue several objects from an HTS ring (multi-consumers safe).
304 * When the requested objects are more than the available objects,
305 * only dequeue the actual number of objects.
308 * A pointer to the ring structure.
310 * A pointer to a table of void * pointers (objects) that will be filled.
312 * The number of objects to dequeue from the ring to the obj_table.
314 * If non-NULL, returns the number of remaining ring entries after the
315 * dequeue has finished.
317 * - n: Actual number of objects dequeued, 0 if ring is empty
320 static __rte_always_inline unsigned int
321 rte_ring_mc_hts_dequeue_burst(struct rte_ring *r, void **obj_table,
322 unsigned int n, unsigned int *available)
324 return rte_ring_mc_hts_dequeue_burst_elem(r, obj_table,
325 sizeof(uintptr_t), n, available);
332 #endif /* _RTE_RING_HTS_H_ */