1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2010-2020 Intel Corporation
4 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
6 * Derived from FreeBSD's bufring.h
7 * Used as BSD-3 Licensed with permission from Kip Macy.
10 #ifndef _RTE_RING_PEEK_H_
11 #define _RTE_RING_PEEK_H_
15 * @b EXPERIMENTAL: this API may change without prior notice
16 * It is not recommended to include this file directly.
17 * Please include <rte_ring_elem.h> instead.
20 * Introduction of rte_ring with serialized producer/consumer (HTS sync mode)
21 * makes possible to split public enqueue/dequeue API into two phases:
22 * - enqueue/dequeue start
23 * - enqueue/dequeue finish
24 * That allows user to inspect objects in the ring without removing them
25 * from it (aka MT safe peek).
26 * Note that right now this new API is available only for two sync modes:
27 * 1) Single Producer/Single Consumer (RTE_RING_SYNC_ST)
28 * 2) Serialized Producer/Serialized Consumer (RTE_RING_SYNC_MT_HTS).
29 * It is a user responsibility to create/init ring with appropriate sync
32 * // read 1 elem from the ring:
33 * n = rte_ring_dequeue_bulk_start(ring, &obj, 1, NULL);
36 * if (object_examine(obj) == KEEP)
37 * //decided to keep it in the ring.
38 * rte_ring_dequeue_finish(ring, 0);
40 * //decided to remove it from the ring.
41 * rte_ring_dequeue_finish(ring, n);
43 * Note that between _start_ and _finish_ none other thread can proceed
44 * with enqueue(/dequeue) operation till _finish_ completes.
51 #include <rte_ring_peek_c11_mem.h>
54 * @internal This function moves prod head value.
56 static __rte_always_inline unsigned int
57 __rte_ring_do_enqueue_start(struct rte_ring *r, uint32_t n,
58 enum rte_ring_queue_behavior behavior, uint32_t *free_space)
60 uint32_t free, head, next;
62 switch (r->prod.sync_type) {
63 case RTE_RING_SYNC_ST:
64 n = __rte_ring_move_prod_head(r, RTE_RING_SYNC_ST, n,
65 behavior, &head, &next, &free);
67 case RTE_RING_SYNC_MT_HTS:
68 n = __rte_ring_hts_move_prod_head(r, n, behavior,
71 case RTE_RING_SYNC_MT:
72 case RTE_RING_SYNC_MT_RTS:
74 /* unsupported mode, shouldn't be here */
79 if (free_space != NULL)
80 *free_space = free - n;
85 * Start to enqueue several objects on the ring.
86 * Note that no actual objects are put in the queue by this function,
87 * it just reserves for user such ability.
88 * User has to call appropriate enqueue_elem_finish() to copy objects into the
89 * queue and complete given enqueue operation.
92 * A pointer to the ring structure.
94 * The number of objects to add in the ring from the obj_table.
96 * if non-NULL, returns the amount of space in the ring after the
97 * enqueue operation has finished.
99 * The number of objects that can be enqueued, either 0 or n
102 static __rte_always_inline unsigned int
103 rte_ring_enqueue_bulk_elem_start(struct rte_ring *r, unsigned int n,
104 unsigned int *free_space)
106 return __rte_ring_do_enqueue_start(r, n, RTE_RING_QUEUE_FIXED,
111 * Start to enqueue several objects on the ring.
112 * Note that no actual objects are put in the queue by this function,
113 * it just reserves for user such ability.
114 * User has to call appropriate enqueue_finish() to copy objects into the
115 * queue and complete given enqueue operation.
118 * A pointer to the ring structure.
120 * The number of objects to add in the ring from the obj_table.
122 * if non-NULL, returns the amount of space in the ring after the
123 * enqueue operation has finished.
125 * The number of objects that can be enqueued, either 0 or n
128 static __rte_always_inline unsigned int
129 rte_ring_enqueue_bulk_start(struct rte_ring *r, unsigned int n,
130 unsigned int *free_space)
132 return rte_ring_enqueue_bulk_elem_start(r, n, free_space);
136 * Start to enqueue several objects on the ring.
137 * Note that no actual objects are put in the queue by this function,
138 * it just reserves for user such ability.
139 * User has to call appropriate enqueue_elem_finish() to copy objects into the
140 * queue and complete given enqueue operation.
143 * A pointer to the ring structure.
145 * The number of objects to add in the ring from the obj_table.
147 * if non-NULL, returns the amount of space in the ring after the
148 * enqueue operation has finished.
150 * Actual number of objects that can be enqueued.
153 static __rte_always_inline unsigned int
154 rte_ring_enqueue_burst_elem_start(struct rte_ring *r, unsigned int n,
155 unsigned int *free_space)
157 return __rte_ring_do_enqueue_start(r, n, RTE_RING_QUEUE_VARIABLE,
162 * Start to enqueue several objects on the ring.
163 * Note that no actual objects are put in the queue by this function,
164 * it just reserves for user such ability.
165 * User has to call appropriate enqueue_finish() to copy objects into the
166 * queue and complete given enqueue operation.
169 * A pointer to the ring structure.
171 * The number of objects to add in the ring from the obj_table.
173 * if non-NULL, returns the amount of space in the ring after the
174 * enqueue operation has finished.
176 * Actual number of objects that can be enqueued.
179 static __rte_always_inline unsigned int
180 rte_ring_enqueue_burst_start(struct rte_ring *r, unsigned int n,
181 unsigned int *free_space)
183 return rte_ring_enqueue_burst_elem_start(r, n, free_space);
187 * Complete to enqueue several objects on the ring.
188 * Note that number of objects to enqueue should not exceed previous
189 * enqueue_start return value.
192 * A pointer to the ring structure.
194 * A pointer to a table of objects.
196 * The size of ring element, in bytes. It must be a multiple of 4.
197 * This must be the same value used while creating the ring. Otherwise
198 * the results are undefined.
200 * The number of objects to add to the ring from the obj_table.
203 static __rte_always_inline void
204 rte_ring_enqueue_elem_finish(struct rte_ring *r, const void *obj_table,
205 unsigned int esize, unsigned int n)
209 switch (r->prod.sync_type) {
210 case RTE_RING_SYNC_ST:
211 n = __rte_ring_st_get_tail(&r->prod, &tail, n);
213 __rte_ring_enqueue_elems(r, tail, obj_table, esize, n);
214 __rte_ring_st_set_head_tail(&r->prod, tail, n, 1);
216 case RTE_RING_SYNC_MT_HTS:
217 n = __rte_ring_hts_get_tail(&r->hts_prod, &tail, n);
219 __rte_ring_enqueue_elems(r, tail, obj_table, esize, n);
220 __rte_ring_hts_set_head_tail(&r->hts_prod, tail, n, 1);
222 case RTE_RING_SYNC_MT:
223 case RTE_RING_SYNC_MT_RTS:
225 /* unsupported mode, shouldn't be here */
231 * Complete to enqueue several objects on the ring.
232 * Note that number of objects to enqueue should not exceed previous
233 * enqueue_start return value.
236 * A pointer to the ring structure.
238 * A pointer to a table of objects.
240 * The number of objects to add to the ring from the obj_table.
243 static __rte_always_inline void
244 rte_ring_enqueue_finish(struct rte_ring *r, void * const *obj_table,
247 rte_ring_enqueue_elem_finish(r, obj_table, sizeof(uintptr_t), n);
251 * @internal This function moves cons head value and copies up to *n*
252 * objects from the ring to the user provided obj_table.
254 static __rte_always_inline unsigned int
255 __rte_ring_do_dequeue_start(struct rte_ring *r, void *obj_table,
256 uint32_t esize, uint32_t n, enum rte_ring_queue_behavior behavior,
259 uint32_t avail, head, next;
261 switch (r->cons.sync_type) {
262 case RTE_RING_SYNC_ST:
263 n = __rte_ring_move_cons_head(r, RTE_RING_SYNC_ST, n,
264 behavior, &head, &next, &avail);
266 case RTE_RING_SYNC_MT_HTS:
267 n = __rte_ring_hts_move_cons_head(r, n, behavior,
270 case RTE_RING_SYNC_MT:
271 case RTE_RING_SYNC_MT_RTS:
273 /* unsupported mode, shouldn't be here */
279 __rte_ring_dequeue_elems(r, head, obj_table, esize, n);
281 if (available != NULL)
282 *available = avail - n;
287 * Start to dequeue several objects from the ring.
288 * Note that user has to call appropriate dequeue_finish()
289 * to complete given dequeue operation and actually remove objects the ring.
292 * A pointer to the ring structure.
294 * A pointer to a table of objects that will be filled.
296 * The size of ring element, in bytes. It must be a multiple of 4.
297 * This must be the same value used while creating the ring. Otherwise
298 * the results are undefined.
300 * The number of objects to dequeue from the ring to the obj_table.
302 * If non-NULL, returns the number of remaining ring entries after the
303 * dequeue has finished.
305 * The number of objects dequeued, either 0 or n.
308 static __rte_always_inline unsigned int
309 rte_ring_dequeue_bulk_elem_start(struct rte_ring *r, void *obj_table,
310 unsigned int esize, unsigned int n, unsigned int *available)
312 return __rte_ring_do_dequeue_start(r, obj_table, esize, n,
313 RTE_RING_QUEUE_FIXED, available);
317 * Start to dequeue several objects from the ring.
318 * Note that user has to call appropriate dequeue_finish()
319 * to complete given dequeue operation and actually remove objects the ring.
322 * A pointer to the ring structure.
324 * A pointer to a table of void * pointers (objects) that will be filled.
326 * The number of objects to dequeue from the ring to the obj_table.
328 * If non-NULL, returns the number of remaining ring entries after the
329 * dequeue has finished.
331 * Actual number of objects dequeued.
334 static __rte_always_inline unsigned int
335 rte_ring_dequeue_bulk_start(struct rte_ring *r, void **obj_table,
336 unsigned int n, unsigned int *available)
338 return rte_ring_dequeue_bulk_elem_start(r, obj_table, sizeof(uintptr_t),
343 * Start to dequeue several objects from the ring.
344 * Note that user has to call appropriate dequeue_finish()
345 * to complete given dequeue operation and actually remove objects the ring.
348 * A pointer to the ring structure.
350 * A pointer to a table of objects that will be filled.
352 * The size of ring element, in bytes. It must be a multiple of 4.
353 * This must be the same value used while creating the ring. Otherwise
354 * the results are undefined.
356 * The number of objects to dequeue from the ring to the obj_table.
358 * If non-NULL, returns the number of remaining ring entries after the
359 * dequeue has finished.
361 * The actual number of objects dequeued.
364 static __rte_always_inline unsigned int
365 rte_ring_dequeue_burst_elem_start(struct rte_ring *r, void *obj_table,
366 unsigned int esize, unsigned int n, unsigned int *available)
368 return __rte_ring_do_dequeue_start(r, obj_table, esize, n,
369 RTE_RING_QUEUE_VARIABLE, available);
373 * Start to dequeue several objects from the ring.
374 * Note that user has to call appropriate dequeue_finish()
375 * to complete given dequeue operation and actually remove objects the ring.
378 * A pointer to the ring structure.
380 * A pointer to a table of void * pointers (objects) that will be filled.
382 * The number of objects to dequeue from the ring to the obj_table.
384 * If non-NULL, returns the number of remaining ring entries after the
385 * dequeue has finished.
387 * The actual number of objects dequeued.
390 static __rte_always_inline unsigned int
391 rte_ring_dequeue_burst_start(struct rte_ring *r, void **obj_table,
392 unsigned int n, unsigned int *available)
394 return rte_ring_dequeue_burst_elem_start(r, obj_table,
395 sizeof(uintptr_t), n, available);
399 * Complete to dequeue several objects from the ring.
400 * Note that number of objects to dequeue should not exceed previous
401 * dequeue_start return value.
404 * A pointer to the ring structure.
406 * The number of objects to remove from the ring.
409 static __rte_always_inline void
410 rte_ring_dequeue_elem_finish(struct rte_ring *r, unsigned int n)
414 switch (r->cons.sync_type) {
415 case RTE_RING_SYNC_ST:
416 n = __rte_ring_st_get_tail(&r->cons, &tail, n);
417 __rte_ring_st_set_head_tail(&r->cons, tail, n, 0);
419 case RTE_RING_SYNC_MT_HTS:
420 n = __rte_ring_hts_get_tail(&r->hts_cons, &tail, n);
421 __rte_ring_hts_set_head_tail(&r->hts_cons, tail, n, 0);
423 case RTE_RING_SYNC_MT:
424 case RTE_RING_SYNC_MT_RTS:
426 /* unsupported mode, shouldn't be here */
432 * Complete to dequeue several objects from the ring.
433 * Note that number of objects to dequeue should not exceed previous
434 * dequeue_start return value.
437 * A pointer to the ring structure.
439 * The number of objects to remove from the ring.
442 static __rte_always_inline void
443 rte_ring_dequeue_finish(struct rte_ring *r, unsigned int n)
445 rte_ring_dequeue_elem_finish(r, n);
452 #endif /* _RTE_RING_PEEK_H_ */