1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2010-2020 Intel Corporation
4 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
6 * Derived from FreeBSD's bufring.h
7 * Used as BSD-3 Licensed with permission from Kip Macy.
10 #ifndef _RTE_RING_PEEK_H_
11 #define _RTE_RING_PEEK_H_
15 * @b EXPERIMENTAL: this API may change without prior notice
16 * It is not recommended to include this file directly.
17 * Please include <rte_ring_elem.h> instead.
20 * Introduction of rte_ring with serialized producer/consumer (HTS sync mode)
21 * makes possible to split public enqueue/dequeue API into two phases:
22 * - enqueue/dequeue start
23 * - enqueue/dequeue finish
24 * That allows user to inspect objects in the ring without removing them
25 * from it (aka MT safe peek).
26 * Note that right now this new API is available only for two sync modes:
27 * 1) Single Producer/Single Consumer (RTE_RING_SYNC_ST)
28 * 2) Serialized Producer/Serialized Consumer (RTE_RING_SYNC_MT_HTS).
29 * It is a user responsibility to create/init ring with appropriate sync
32 * // read 1 elem from the ring:
33 * n = rte_ring_dequeue_bulk_start(ring, &obj, 1, NULL);
36 * if (object_examine(obj) == KEEP)
37 * //decided to keep it in the ring.
38 * rte_ring_dequeue_finish(ring, 0);
40 * //decided to remove it from the ring.
41 * rte_ring_dequeue_finish(ring, n);
43 * Note that between _start_ and _finish_ none other thread can proceed
44 * with enqueue(/dequeue) operation till _finish_ completes.
51 #include <rte_ring_peek_c11_mem.h>
54 * @internal This function moves prod head value.
56 static __rte_always_inline unsigned int
57 __rte_ring_do_enqueue_start(struct rte_ring *r, uint32_t n,
58 enum rte_ring_queue_behavior behavior, uint32_t *free_space)
60 uint32_t free, head, next;
62 switch (r->prod.sync_type) {
63 case RTE_RING_SYNC_ST:
64 n = __rte_ring_move_prod_head(r, RTE_RING_SYNC_ST, n,
65 behavior, &head, &next, &free);
67 case RTE_RING_SYNC_MT_HTS:
68 n = __rte_ring_hts_move_prod_head(r, n, behavior,
72 /* unsupported mode, shouldn't be here */
77 if (free_space != NULL)
78 *free_space = free - n;
83 * Start to enqueue several objects on the ring.
84 * Note that no actual objects are put in the queue by this function,
85 * it just reserves for user such ability.
86 * User has to call appropriate enqueue_elem_finish() to copy objects into the
87 * queue and complete given enqueue operation.
90 * A pointer to the ring structure.
92 * The number of objects to add in the ring from the obj_table.
94 * if non-NULL, returns the amount of space in the ring after the
95 * enqueue operation has finished.
97 * The number of objects that can be enqueued, either 0 or n
100 static __rte_always_inline unsigned int
101 rte_ring_enqueue_bulk_elem_start(struct rte_ring *r, unsigned int n,
102 unsigned int *free_space)
104 return __rte_ring_do_enqueue_start(r, n, RTE_RING_QUEUE_FIXED,
109 * Start to enqueue several objects on the ring.
110 * Note that no actual objects are put in the queue by this function,
111 * it just reserves for user such ability.
112 * User has to call appropriate enqueue_finish() to copy objects into the
113 * queue and complete given enqueue operation.
116 * A pointer to the ring structure.
118 * The number of objects to add in the ring from the obj_table.
120 * if non-NULL, returns the amount of space in the ring after the
121 * enqueue operation has finished.
123 * The number of objects that can be enqueued, either 0 or n
126 static __rte_always_inline unsigned int
127 rte_ring_enqueue_bulk_start(struct rte_ring *r, unsigned int n,
128 unsigned int *free_space)
130 return rte_ring_enqueue_bulk_elem_start(r, n, free_space);
134 * Start to enqueue several objects on the ring.
135 * Note that no actual objects are put in the queue by this function,
136 * it just reserves for user such ability.
137 * User has to call appropriate enqueue_elem_finish() to copy objects into the
138 * queue and complete given enqueue operation.
141 * A pointer to the ring structure.
143 * The number of objects to add in the ring from the obj_table.
145 * if non-NULL, returns the amount of space in the ring after the
146 * enqueue operation has finished.
148 * Actual number of objects that can be enqueued.
151 static __rte_always_inline unsigned int
152 rte_ring_enqueue_burst_elem_start(struct rte_ring *r, unsigned int n,
153 unsigned int *free_space)
155 return __rte_ring_do_enqueue_start(r, n, RTE_RING_QUEUE_VARIABLE,
160 * Start to enqueue several objects on the ring.
161 * Note that no actual objects are put in the queue by this function,
162 * it just reserves for user such ability.
163 * User has to call appropriate enqueue_finish() to copy objects into the
164 * queue and complete given enqueue operation.
167 * A pointer to the ring structure.
169 * The number of objects to add in the ring from the obj_table.
171 * if non-NULL, returns the amount of space in the ring after the
172 * enqueue operation has finished.
174 * Actual number of objects that can be enqueued.
177 static __rte_always_inline unsigned int
178 rte_ring_enqueue_burst_start(struct rte_ring *r, unsigned int n,
179 unsigned int *free_space)
181 return rte_ring_enqueue_burst_elem_start(r, n, free_space);
185 * Complete to enqueue several objects on the ring.
186 * Note that number of objects to enqueue should not exceed previous
187 * enqueue_start return value.
190 * A pointer to the ring structure.
192 * A pointer to a table of objects.
194 * The size of ring element, in bytes. It must be a multiple of 4.
195 * This must be the same value used while creating the ring. Otherwise
196 * the results are undefined.
198 * The number of objects to add to the ring from the obj_table.
201 static __rte_always_inline void
202 rte_ring_enqueue_elem_finish(struct rte_ring *r, const void *obj_table,
203 unsigned int esize, unsigned int n)
207 switch (r->prod.sync_type) {
208 case RTE_RING_SYNC_ST:
209 n = __rte_ring_st_get_tail(&r->prod, &tail, n);
211 __rte_ring_enqueue_elems(r, tail, obj_table, esize, n);
212 __rte_ring_st_set_head_tail(&r->prod, tail, n, 1);
214 case RTE_RING_SYNC_MT_HTS:
215 n = __rte_ring_hts_get_tail(&r->hts_prod, &tail, n);
217 __rte_ring_enqueue_elems(r, tail, obj_table, esize, n);
218 __rte_ring_hts_set_head_tail(&r->hts_prod, tail, n, 1);
221 /* unsupported mode, shouldn't be here */
227 * Complete to enqueue several objects on the ring.
228 * Note that number of objects to enqueue should not exceed previous
229 * enqueue_start return value.
232 * A pointer to the ring structure.
234 * A pointer to a table of objects.
236 * The number of objects to add to the ring from the obj_table.
239 static __rte_always_inline void
240 rte_ring_enqueue_finish(struct rte_ring *r, void * const *obj_table,
243 rte_ring_enqueue_elem_finish(r, obj_table, sizeof(uintptr_t), n);
247 * @internal This function moves cons head value and copies up to *n*
248 * objects from the ring to the user provided obj_table.
250 static __rte_always_inline unsigned int
251 __rte_ring_do_dequeue_start(struct rte_ring *r, void *obj_table,
252 uint32_t esize, uint32_t n, enum rte_ring_queue_behavior behavior,
255 uint32_t avail, head, next;
257 switch (r->cons.sync_type) {
258 case RTE_RING_SYNC_ST:
259 n = __rte_ring_move_cons_head(r, RTE_RING_SYNC_ST, n,
260 behavior, &head, &next, &avail);
262 case RTE_RING_SYNC_MT_HTS:
263 n = __rte_ring_hts_move_cons_head(r, n, behavior,
267 /* unsupported mode, shouldn't be here */
273 __rte_ring_dequeue_elems(r, head, obj_table, esize, n);
275 if (available != NULL)
276 *available = avail - n;
281 * Start to dequeue several objects from the ring.
282 * Note that user has to call appropriate dequeue_finish()
283 * to complete given dequeue operation and actually remove objects the ring.
286 * A pointer to the ring structure.
288 * A pointer to a table of objects that will be filled.
290 * The size of ring element, in bytes. It must be a multiple of 4.
291 * This must be the same value used while creating the ring. Otherwise
292 * the results are undefined.
294 * The number of objects to dequeue from the ring to the obj_table.
296 * If non-NULL, returns the number of remaining ring entries after the
297 * dequeue has finished.
299 * The number of objects dequeued, either 0 or n.
302 static __rte_always_inline unsigned int
303 rte_ring_dequeue_bulk_elem_start(struct rte_ring *r, void *obj_table,
304 unsigned int esize, unsigned int n, unsigned int *available)
306 return __rte_ring_do_dequeue_start(r, obj_table, esize, n,
307 RTE_RING_QUEUE_FIXED, available);
311 * Start to dequeue several objects from the ring.
312 * Note that user has to call appropriate dequeue_finish()
313 * to complete given dequeue operation and actually remove objects the ring.
316 * A pointer to the ring structure.
318 * A pointer to a table of void * pointers (objects) that will be filled.
320 * The number of objects to dequeue from the ring to the obj_table.
322 * If non-NULL, returns the number of remaining ring entries after the
323 * dequeue has finished.
325 * Actual number of objects dequeued.
328 static __rte_always_inline unsigned int
329 rte_ring_dequeue_bulk_start(struct rte_ring *r, void **obj_table,
330 unsigned int n, unsigned int *available)
332 return rte_ring_dequeue_bulk_elem_start(r, obj_table, sizeof(uintptr_t),
337 * Start to dequeue several objects from the ring.
338 * Note that user has to call appropriate dequeue_finish()
339 * to complete given dequeue operation and actually remove objects the ring.
342 * A pointer to the ring structure.
344 * A pointer to a table of objects that will be filled.
346 * The size of ring element, in bytes. It must be a multiple of 4.
347 * This must be the same value used while creating the ring. Otherwise
348 * the results are undefined.
350 * The number of objects to dequeue from the ring to the obj_table.
352 * If non-NULL, returns the number of remaining ring entries after the
353 * dequeue has finished.
355 * The actual number of objects dequeued.
358 static __rte_always_inline unsigned int
359 rte_ring_dequeue_burst_elem_start(struct rte_ring *r, void *obj_table,
360 unsigned int esize, unsigned int n, unsigned int *available)
362 return __rte_ring_do_dequeue_start(r, obj_table, esize, n,
363 RTE_RING_QUEUE_VARIABLE, available);
367 * Start to dequeue several objects from the ring.
368 * Note that user has to call appropriate dequeue_finish()
369 * to complete given dequeue operation and actually remove objects the ring.
372 * A pointer to the ring structure.
374 * A pointer to a table of void * pointers (objects) that will be filled.
376 * The number of objects to dequeue from the ring to the obj_table.
378 * If non-NULL, returns the number of remaining ring entries after the
379 * dequeue has finished.
381 * The actual number of objects dequeued.
384 static __rte_always_inline unsigned int
385 rte_ring_dequeue_burst_start(struct rte_ring *r, void **obj_table,
386 unsigned int n, unsigned int *available)
388 return rte_ring_dequeue_burst_elem_start(r, obj_table,
389 sizeof(uintptr_t), n, available);
393 * Complete to dequeue several objects from the ring.
394 * Note that number of objects to dequeue should not exceed previous
395 * dequeue_start return value.
398 * A pointer to the ring structure.
400 * The number of objects to remove from the ring.
403 static __rte_always_inline void
404 rte_ring_dequeue_elem_finish(struct rte_ring *r, unsigned int n)
408 switch (r->cons.sync_type) {
409 case RTE_RING_SYNC_ST:
410 n = __rte_ring_st_get_tail(&r->cons, &tail, n);
411 __rte_ring_st_set_head_tail(&r->cons, tail, n, 0);
413 case RTE_RING_SYNC_MT_HTS:
414 n = __rte_ring_hts_get_tail(&r->hts_cons, &tail, n);
415 __rte_ring_hts_set_head_tail(&r->hts_cons, tail, n, 0);
418 /* unsupported mode, shouldn't be here */
424 * Complete to dequeue several objects from the ring.
425 * Note that number of objects to dequeue should not exceed previous
426 * dequeue_start return value.
429 * A pointer to the ring structure.
431 * The number of objects to remove from the ring.
434 static __rte_always_inline void
435 rte_ring_dequeue_finish(struct rte_ring *r, unsigned int n)
437 rte_ring_dequeue_elem_finish(r, n);
444 #endif /* _RTE_RING_PEEK_H_ */