1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2019 Arm Limited
4 * Copyright (c) 2010-2017 Intel Corporation
5 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
7 * Derived from FreeBSD's bufring.h
8 * Used as BSD-3 Licensed with permission from Kip Macy.
11 #ifndef _RTE_RING_ELEM_H_
12 #define _RTE_RING_ELEM_H_
16 * RTE Ring with user defined element size
23 #include <rte_ring_core.h>
24 #include <rte_ring_elem_pvt.h>
27 * Calculate the memory size needed for a ring with given element size
29 * This function returns the number of bytes needed for a ring, given
30 * the number of elements in it and the size of the element. This value
31 * is the sum of the size of the structure rte_ring and the size of the
32 * memory needed for storing the elements. The value is aligned to a cache
36 * The size of ring element, in bytes. It must be a multiple of 4.
38 * The number of elements in the ring (must be a power of 2).
40 * - The memory size needed for the ring on success.
41 * - -EINVAL - esize is not a multiple of 4 or count provided is not a
44 ssize_t rte_ring_get_memsize_elem(unsigned int esize, unsigned int count);
47 * Create a new ring named *name* that stores elements with given size.
49 * This function uses ``memzone_reserve()`` to allocate memory. Then it
50 * calls rte_ring_init() to initialize an empty ring.
52 * The new ring size is set to *count*, which must be a power of
53 * two. Water marking is disabled by default. The real usable ring size
54 * is *count-1* instead of *count* to differentiate a full ring from an
57 * The ring is added in RTE_TAILQ_RING list.
60 * The name of the ring.
62 * The size of ring element, in bytes. It must be a multiple of 4.
64 * The number of elements in the ring (must be a power of 2).
66 * The *socket_id* argument is the socket identifier in case of
67 * NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
68 * constraint for the reserved zone.
70 * An OR of the following:
71 * - One of mutually exclusive flags that define producer behavior:
72 * - RING_F_SP_ENQ: If this flag is set, the default behavior when
73 * using ``rte_ring_enqueue()`` or ``rte_ring_enqueue_bulk()``
74 * is "single-producer".
75 * - RING_F_MP_RTS_ENQ: If this flag is set, the default behavior when
76 * using ``rte_ring_enqueue()`` or ``rte_ring_enqueue_bulk()``
77 * is "multi-producer RTS mode".
78 * - RING_F_MP_HTS_ENQ: If this flag is set, the default behavior when
79 * using ``rte_ring_enqueue()`` or ``rte_ring_enqueue_bulk()``
80 * is "multi-producer HTS mode".
81 * If none of these flags is set, then default "multi-producer"
82 * behavior is selected.
83 * - One of mutually exclusive flags that define consumer behavior:
84 * - RING_F_SC_DEQ: If this flag is set, the default behavior when
85 * using ``rte_ring_dequeue()`` or ``rte_ring_dequeue_bulk()``
86 * is "single-consumer". Otherwise, it is "multi-consumers".
87 * - RING_F_MC_RTS_DEQ: If this flag is set, the default behavior when
88 * using ``rte_ring_dequeue()`` or ``rte_ring_dequeue_bulk()``
89 * is "multi-consumer RTS mode".
90 * - RING_F_MC_HTS_DEQ: If this flag is set, the default behavior when
91 * using ``rte_ring_dequeue()`` or ``rte_ring_dequeue_bulk()``
92 * is "multi-consumer HTS mode".
93 * If none of these flags is set, then default "multi-consumer"
94 * behavior is selected.
96 * On success, the pointer to the new allocated ring. NULL on error with
97 * rte_errno set appropriately. Possible errno values include:
98 * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
99 * - E_RTE_SECONDARY - function was called from a secondary process instance
100 * - EINVAL - esize is not a multiple of 4 or count provided is not a
102 * - ENOSPC - the maximum number of memzones has already been allocated
103 * - EEXIST - a memzone with the same name already exists
104 * - ENOMEM - no appropriate memory area found in which to create memzone
106 struct rte_ring *rte_ring_create_elem(const char *name, unsigned int esize,
107 unsigned int count, int socket_id, unsigned int flags);
110 * Enqueue several objects on the ring (multi-producers safe).
112 * This function uses a "compare and set" instruction to move the
113 * producer index atomically.
116 * A pointer to the ring structure.
118 * A pointer to a table of objects.
120 * The size of ring element, in bytes. It must be a multiple of 4.
121 * This must be the same value used while creating the ring. Otherwise
122 * the results are undefined.
124 * The number of objects to add in the ring from the obj_table.
126 * if non-NULL, returns the amount of space in the ring after the
127 * enqueue operation has finished.
129 * The number of objects enqueued, either 0 or n
131 static __rte_always_inline unsigned int
132 rte_ring_mp_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table,
133 unsigned int esize, unsigned int n, unsigned int *free_space)
135 return __rte_ring_do_enqueue_elem(r, obj_table, esize, n,
136 RTE_RING_QUEUE_FIXED, RTE_RING_SYNC_MT, free_space);
140 * Enqueue several objects on a ring
142 * @warning This API is NOT multi-producers safe
145 * A pointer to the ring structure.
147 * A pointer to a table of objects.
149 * The size of ring element, in bytes. It must be a multiple of 4.
150 * This must be the same value used while creating the ring. Otherwise
151 * the results are undefined.
153 * The number of objects to add in the ring from the obj_table.
155 * if non-NULL, returns the amount of space in the ring after the
156 * enqueue operation has finished.
158 * The number of objects enqueued, either 0 or n
160 static __rte_always_inline unsigned int
161 rte_ring_sp_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table,
162 unsigned int esize, unsigned int n, unsigned int *free_space)
164 return __rte_ring_do_enqueue_elem(r, obj_table, esize, n,
165 RTE_RING_QUEUE_FIXED, RTE_RING_SYNC_ST, free_space);
168 #include <rte_ring_hts.h>
169 #include <rte_ring_rts.h>
172 * Enqueue several objects on a ring.
174 * This function calls the multi-producer or the single-producer
175 * version depending on the default behavior that was specified at
176 * ring creation time (see flags).
179 * A pointer to the ring structure.
181 * A pointer to a table of objects.
183 * The size of ring element, in bytes. It must be a multiple of 4.
184 * This must be the same value used while creating the ring. Otherwise
185 * the results are undefined.
187 * The number of objects to add in the ring from the obj_table.
189 * if non-NULL, returns the amount of space in the ring after the
190 * enqueue operation has finished.
192 * The number of objects enqueued, either 0 or n
194 static __rte_always_inline unsigned int
195 rte_ring_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table,
196 unsigned int esize, unsigned int n, unsigned int *free_space)
198 switch (r->prod.sync_type) {
199 case RTE_RING_SYNC_MT:
200 return rte_ring_mp_enqueue_bulk_elem(r, obj_table, esize, n,
202 case RTE_RING_SYNC_ST:
203 return rte_ring_sp_enqueue_bulk_elem(r, obj_table, esize, n,
205 case RTE_RING_SYNC_MT_RTS:
206 return rte_ring_mp_rts_enqueue_bulk_elem(r, obj_table, esize, n,
208 case RTE_RING_SYNC_MT_HTS:
209 return rte_ring_mp_hts_enqueue_bulk_elem(r, obj_table, esize, n,
213 /* valid ring should never reach this point */
215 if (free_space != NULL)
221 * Enqueue one object on a ring (multi-producers safe).
223 * This function uses a "compare and set" instruction to move the
224 * producer index atomically.
227 * A pointer to the ring structure.
229 * A pointer to the object to be added.
231 * The size of ring element, in bytes. It must be a multiple of 4.
232 * This must be the same value used while creating the ring. Otherwise
233 * the results are undefined.
235 * - 0: Success; objects enqueued.
236 * - -ENOBUFS: Not enough room in the ring to enqueue; no object is enqueued.
238 static __rte_always_inline int
239 rte_ring_mp_enqueue_elem(struct rte_ring *r, void *obj, unsigned int esize)
241 return rte_ring_mp_enqueue_bulk_elem(r, obj, esize, 1, NULL) ? 0 :
246 * Enqueue one object on a ring
248 * @warning This API is NOT multi-producers safe
251 * A pointer to the ring structure.
253 * A pointer to the object to be added.
255 * The size of ring element, in bytes. It must be a multiple of 4.
256 * This must be the same value used while creating the ring. Otherwise
257 * the results are undefined.
259 * - 0: Success; objects enqueued.
260 * - -ENOBUFS: Not enough room in the ring to enqueue; no object is enqueued.
262 static __rte_always_inline int
263 rte_ring_sp_enqueue_elem(struct rte_ring *r, void *obj, unsigned int esize)
265 return rte_ring_sp_enqueue_bulk_elem(r, obj, esize, 1, NULL) ? 0 :
270 * Enqueue one object on a ring.
272 * This function calls the multi-producer or the single-producer
273 * version, depending on the default behaviour that was specified at
274 * ring creation time (see flags).
277 * A pointer to the ring structure.
279 * A pointer to the object to be added.
281 * The size of ring element, in bytes. It must be a multiple of 4.
282 * This must be the same value used while creating the ring. Otherwise
283 * the results are undefined.
285 * - 0: Success; objects enqueued.
286 * - -ENOBUFS: Not enough room in the ring to enqueue; no object is enqueued.
288 static __rte_always_inline int
289 rte_ring_enqueue_elem(struct rte_ring *r, void *obj, unsigned int esize)
291 return rte_ring_enqueue_bulk_elem(r, obj, esize, 1, NULL) ? 0 :
296 * Dequeue several objects from a ring (multi-consumers safe).
298 * This function uses a "compare and set" instruction to move the
299 * consumer index atomically.
302 * A pointer to the ring structure.
304 * A pointer to a table of objects that will be filled.
306 * The size of ring element, in bytes. It must be a multiple of 4.
307 * This must be the same value used while creating the ring. Otherwise
308 * the results are undefined.
310 * The number of objects to dequeue from the ring to the obj_table.
312 * If non-NULL, returns the number of remaining ring entries after the
313 * dequeue has finished.
315 * The number of objects dequeued, either 0 or n
317 static __rte_always_inline unsigned int
318 rte_ring_mc_dequeue_bulk_elem(struct rte_ring *r, void *obj_table,
319 unsigned int esize, unsigned int n, unsigned int *available)
321 return __rte_ring_do_dequeue_elem(r, obj_table, esize, n,
322 RTE_RING_QUEUE_FIXED, RTE_RING_SYNC_MT, available);
326 * Dequeue several objects from a ring (NOT multi-consumers safe).
329 * A pointer to the ring structure.
331 * A pointer to a table of objects that will be filled.
333 * The size of ring element, in bytes. It must be a multiple of 4.
334 * This must be the same value used while creating the ring. Otherwise
335 * the results are undefined.
337 * The number of objects to dequeue from the ring to the obj_table,
338 * must be strictly positive.
340 * If non-NULL, returns the number of remaining ring entries after the
341 * dequeue has finished.
343 * The number of objects dequeued, either 0 or n
345 static __rte_always_inline unsigned int
346 rte_ring_sc_dequeue_bulk_elem(struct rte_ring *r, void *obj_table,
347 unsigned int esize, unsigned int n, unsigned int *available)
349 return __rte_ring_do_dequeue_elem(r, obj_table, esize, n,
350 RTE_RING_QUEUE_FIXED, RTE_RING_SYNC_ST, available);
354 * Dequeue several objects from a ring.
356 * This function calls the multi-consumers or the single-consumer
357 * version, depending on the default behaviour that was specified at
358 * ring creation time (see flags).
361 * A pointer to the ring structure.
363 * A pointer to a table of objects that will be filled.
365 * The size of ring element, in bytes. It must be a multiple of 4.
366 * This must be the same value used while creating the ring. Otherwise
367 * the results are undefined.
369 * The number of objects to dequeue from the ring to the obj_table.
371 * If non-NULL, returns the number of remaining ring entries after the
372 * dequeue has finished.
374 * The number of objects dequeued, either 0 or n
376 static __rte_always_inline unsigned int
377 rte_ring_dequeue_bulk_elem(struct rte_ring *r, void *obj_table,
378 unsigned int esize, unsigned int n, unsigned int *available)
380 switch (r->cons.sync_type) {
381 case RTE_RING_SYNC_MT:
382 return rte_ring_mc_dequeue_bulk_elem(r, obj_table, esize, n,
384 case RTE_RING_SYNC_ST:
385 return rte_ring_sc_dequeue_bulk_elem(r, obj_table, esize, n,
387 case RTE_RING_SYNC_MT_RTS:
388 return rte_ring_mc_rts_dequeue_bulk_elem(r, obj_table, esize,
390 case RTE_RING_SYNC_MT_HTS:
391 return rte_ring_mc_hts_dequeue_bulk_elem(r, obj_table, esize,
395 /* valid ring should never reach this point */
397 if (available != NULL)
403 * Dequeue one object from a ring (multi-consumers safe).
405 * This function uses a "compare and set" instruction to move the
406 * consumer index atomically.
409 * A pointer to the ring structure.
411 * A pointer to the object that will be filled.
413 * The size of ring element, in bytes. It must be a multiple of 4.
414 * This must be the same value used while creating the ring. Otherwise
415 * the results are undefined.
417 * - 0: Success; objects dequeued.
418 * - -ENOENT: Not enough entries in the ring to dequeue; no object is
421 static __rte_always_inline int
422 rte_ring_mc_dequeue_elem(struct rte_ring *r, void *obj_p,
425 return rte_ring_mc_dequeue_bulk_elem(r, obj_p, esize, 1, NULL) ? 0 :
430 * Dequeue one object from a ring (NOT multi-consumers safe).
433 * A pointer to the ring structure.
435 * A pointer to the object that will be filled.
437 * The size of ring element, in bytes. It must be a multiple of 4.
438 * This must be the same value used while creating the ring. Otherwise
439 * the results are undefined.
441 * - 0: Success; objects dequeued.
442 * - -ENOENT: Not enough entries in the ring to dequeue, no object is
445 static __rte_always_inline int
446 rte_ring_sc_dequeue_elem(struct rte_ring *r, void *obj_p,
449 return rte_ring_sc_dequeue_bulk_elem(r, obj_p, esize, 1, NULL) ? 0 :
454 * Dequeue one object from a ring.
456 * This function calls the multi-consumers or the single-consumer
457 * version depending on the default behaviour that was specified at
458 * ring creation time (see flags).
461 * A pointer to the ring structure.
463 * A pointer to the object that will be filled.
465 * The size of ring element, in bytes. It must be a multiple of 4.
466 * This must be the same value used while creating the ring. Otherwise
467 * the results are undefined.
469 * - 0: Success, objects dequeued.
470 * - -ENOENT: Not enough entries in the ring to dequeue, no object is
473 static __rte_always_inline int
474 rte_ring_dequeue_elem(struct rte_ring *r, void *obj_p, unsigned int esize)
476 return rte_ring_dequeue_bulk_elem(r, obj_p, esize, 1, NULL) ? 0 :
481 * Enqueue several objects on the ring (multi-producers safe).
483 * This function uses a "compare and set" instruction to move the
484 * producer index atomically.
487 * A pointer to the ring structure.
489 * A pointer to a table of objects.
491 * The size of ring element, in bytes. It must be a multiple of 4.
492 * This must be the same value used while creating the ring. Otherwise
493 * the results are undefined.
495 * The number of objects to add in the ring from the obj_table.
497 * if non-NULL, returns the amount of space in the ring after the
498 * enqueue operation has finished.
500 * - n: Actual number of objects enqueued.
502 static __rte_always_inline unsigned int
503 rte_ring_mp_enqueue_burst_elem(struct rte_ring *r, const void *obj_table,
504 unsigned int esize, unsigned int n, unsigned int *free_space)
506 return __rte_ring_do_enqueue_elem(r, obj_table, esize, n,
507 RTE_RING_QUEUE_VARIABLE, RTE_RING_SYNC_MT, free_space);
511 * Enqueue several objects on a ring
513 * @warning This API is NOT multi-producers safe
516 * A pointer to the ring structure.
518 * A pointer to a table of objects.
520 * The size of ring element, in bytes. It must be a multiple of 4.
521 * This must be the same value used while creating the ring. Otherwise
522 * the results are undefined.
524 * The number of objects to add in the ring from the obj_table.
526 * if non-NULL, returns the amount of space in the ring after the
527 * enqueue operation has finished.
529 * - n: Actual number of objects enqueued.
531 static __rte_always_inline unsigned int
532 rte_ring_sp_enqueue_burst_elem(struct rte_ring *r, const void *obj_table,
533 unsigned int esize, unsigned int n, unsigned int *free_space)
535 return __rte_ring_do_enqueue_elem(r, obj_table, esize, n,
536 RTE_RING_QUEUE_VARIABLE, RTE_RING_SYNC_ST, free_space);
540 * Enqueue several objects on a ring.
542 * This function calls the multi-producer or the single-producer
543 * version depending on the default behavior that was specified at
544 * ring creation time (see flags).
547 * A pointer to the ring structure.
549 * A pointer to a table of objects.
551 * The size of ring element, in bytes. It must be a multiple of 4.
552 * This must be the same value used while creating the ring. Otherwise
553 * the results are undefined.
555 * The number of objects to add in the ring from the obj_table.
557 * if non-NULL, returns the amount of space in the ring after the
558 * enqueue operation has finished.
560 * - n: Actual number of objects enqueued.
562 static __rte_always_inline unsigned int
563 rte_ring_enqueue_burst_elem(struct rte_ring *r, const void *obj_table,
564 unsigned int esize, unsigned int n, unsigned int *free_space)
566 switch (r->prod.sync_type) {
567 case RTE_RING_SYNC_MT:
568 return rte_ring_mp_enqueue_burst_elem(r, obj_table, esize, n,
570 case RTE_RING_SYNC_ST:
571 return rte_ring_sp_enqueue_burst_elem(r, obj_table, esize, n,
573 case RTE_RING_SYNC_MT_RTS:
574 return rte_ring_mp_rts_enqueue_burst_elem(r, obj_table, esize,
576 case RTE_RING_SYNC_MT_HTS:
577 return rte_ring_mp_hts_enqueue_burst_elem(r, obj_table, esize,
581 /* valid ring should never reach this point */
583 if (free_space != NULL)
589 * Dequeue several objects from a ring (multi-consumers safe). When the request
590 * objects are more than the available objects, only dequeue the actual number
593 * This function uses a "compare and set" instruction to move the
594 * consumer index atomically.
597 * A pointer to the ring structure.
599 * A pointer to a table of objects that will be filled.
601 * The size of ring element, in bytes. It must be a multiple of 4.
602 * This must be the same value used while creating the ring. Otherwise
603 * the results are undefined.
605 * The number of objects to dequeue from the ring to the obj_table.
607 * If non-NULL, returns the number of remaining ring entries after the
608 * dequeue has finished.
610 * - n: Actual number of objects dequeued, 0 if ring is empty
612 static __rte_always_inline unsigned int
613 rte_ring_mc_dequeue_burst_elem(struct rte_ring *r, void *obj_table,
614 unsigned int esize, unsigned int n, unsigned int *available)
616 return __rte_ring_do_dequeue_elem(r, obj_table, esize, n,
617 RTE_RING_QUEUE_VARIABLE, RTE_RING_SYNC_MT, available);
621 * Dequeue several objects from a ring (NOT multi-consumers safe).When the
622 * request objects are more than the available objects, only dequeue the
623 * actual number of objects
626 * A pointer to the ring structure.
628 * A pointer to a table of objects that will be filled.
630 * The size of ring element, in bytes. It must be a multiple of 4.
631 * This must be the same value used while creating the ring. Otherwise
632 * the results are undefined.
634 * The number of objects to dequeue from the ring to the obj_table.
636 * If non-NULL, returns the number of remaining ring entries after the
637 * dequeue has finished.
639 * - n: Actual number of objects dequeued, 0 if ring is empty
641 static __rte_always_inline unsigned int
642 rte_ring_sc_dequeue_burst_elem(struct rte_ring *r, void *obj_table,
643 unsigned int esize, unsigned int n, unsigned int *available)
645 return __rte_ring_do_dequeue_elem(r, obj_table, esize, n,
646 RTE_RING_QUEUE_VARIABLE, RTE_RING_SYNC_ST, available);
650 * Dequeue multiple objects from a ring up to a maximum number.
652 * This function calls the multi-consumers or the single-consumer
653 * version, depending on the default behaviour that was specified at
654 * ring creation time (see flags).
657 * A pointer to the ring structure.
659 * A pointer to a table of objects that will be filled.
661 * The size of ring element, in bytes. It must be a multiple of 4.
662 * This must be the same value used while creating the ring. Otherwise
663 * the results are undefined.
665 * The number of objects to dequeue from the ring to the obj_table.
667 * If non-NULL, returns the number of remaining ring entries after the
668 * dequeue has finished.
670 * - Number of objects dequeued
672 static __rte_always_inline unsigned int
673 rte_ring_dequeue_burst_elem(struct rte_ring *r, void *obj_table,
674 unsigned int esize, unsigned int n, unsigned int *available)
676 switch (r->cons.sync_type) {
677 case RTE_RING_SYNC_MT:
678 return rte_ring_mc_dequeue_burst_elem(r, obj_table, esize, n,
680 case RTE_RING_SYNC_ST:
681 return rte_ring_sc_dequeue_burst_elem(r, obj_table, esize, n,
683 case RTE_RING_SYNC_MT_RTS:
684 return rte_ring_mc_rts_dequeue_burst_elem(r, obj_table, esize,
686 case RTE_RING_SYNC_MT_HTS:
687 return rte_ring_mc_hts_dequeue_burst_elem(r, obj_table, esize,
691 /* valid ring should never reach this point */
693 if (available != NULL)
698 #include <rte_ring_peek.h>
699 #include <rte_ring_peek_zc.h>
701 #include <rte_ring.h>
707 #endif /* _RTE_RING_ELEM_H_ */