1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2019 Arm Limited
4 * Copyright (c) 2010-2017 Intel Corporation
5 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
7 * Derived from FreeBSD's bufring.h
8 * Used as BSD-3 Licensed with permission from Kip Macy.
11 #ifndef _RTE_RING_ELEM_H_
12 #define _RTE_RING_ELEM_H_
16 * RTE Ring with user defined element size
23 #include <rte_ring_core.h>
26 * Calculate the memory size needed for a ring with given element size
28 * This function returns the number of bytes needed for a ring, given
29 * the number of elements in it and the size of the element. This value
30 * is the sum of the size of the structure rte_ring and the size of the
31 * memory needed for storing the elements. The value is aligned to a cache
35 * The size of ring element, in bytes. It must be a multiple of 4.
37 * The number of elements in the ring (must be a power of 2).
39 * - The memory size needed for the ring on success.
40 * - -EINVAL - esize is not a multiple of 4 or count provided is not a
43 ssize_t rte_ring_get_memsize_elem(unsigned int esize, unsigned int count);
46 * Create a new ring named *name* that stores elements with given size.
48 * This function uses ``memzone_reserve()`` to allocate memory. Then it
49 * calls rte_ring_init() to initialize an empty ring.
51 * The new ring size is set to *count*, which must be a power of
52 * two. Water marking is disabled by default. The real usable ring size
53 * is *count-1* instead of *count* to differentiate a free ring from an
56 * The ring is added in RTE_TAILQ_RING list.
59 * The name of the ring.
61 * The size of ring element, in bytes. It must be a multiple of 4.
63 * The number of elements in the ring (must be a power of 2).
65 * The *socket_id* argument is the socket identifier in case of
66 * NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
67 * constraint for the reserved zone.
69 * An OR of the following:
70 * - One of mutually exclusive flags that define producer behavior:
71 * - RING_F_SP_ENQ: If this flag is set, the default behavior when
72 * using ``rte_ring_enqueue()`` or ``rte_ring_enqueue_bulk()``
73 * is "single-producer".
74 * - RING_F_MP_RTS_ENQ: If this flag is set, the default behavior when
75 * using ``rte_ring_enqueue()`` or ``rte_ring_enqueue_bulk()``
76 * is "multi-producer RTS mode".
77 * - RING_F_MP_HTS_ENQ: If this flag is set, the default behavior when
78 * using ``rte_ring_enqueue()`` or ``rte_ring_enqueue_bulk()``
79 * is "multi-producer HTS mode".
80 * If none of these flags is set, then default "multi-producer"
81 * behavior is selected.
82 * - One of mutually exclusive flags that define consumer behavior:
83 * - RING_F_SC_DEQ: If this flag is set, the default behavior when
84 * using ``rte_ring_dequeue()`` or ``rte_ring_dequeue_bulk()``
85 * is "single-consumer". Otherwise, it is "multi-consumers".
86 * - RING_F_MC_RTS_DEQ: If this flag is set, the default behavior when
87 * using ``rte_ring_dequeue()`` or ``rte_ring_dequeue_bulk()``
88 * is "multi-consumer RTS mode".
89 * - RING_F_MC_HTS_DEQ: If this flag is set, the default behavior when
90 * using ``rte_ring_dequeue()`` or ``rte_ring_dequeue_bulk()``
91 * is "multi-consumer HTS mode".
92 * If none of these flags is set, then default "multi-consumer"
93 * behavior is selected.
95 * On success, the pointer to the new allocated ring. NULL on error with
96 * rte_errno set appropriately. Possible errno values include:
97 * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
98 * - E_RTE_SECONDARY - function was called from a secondary process instance
99 * - EINVAL - esize is not a multiple of 4 or count provided is not a
101 * - ENOSPC - the maximum number of memzones has already been allocated
102 * - EEXIST - a memzone with the same name already exists
103 * - ENOMEM - no appropriate memory area found in which to create memzone
105 struct rte_ring *rte_ring_create_elem(const char *name, unsigned int esize,
106 unsigned int count, int socket_id, unsigned int flags);
108 static __rte_always_inline void
109 __rte_ring_enqueue_elems_32(struct rte_ring *r, const uint32_t size,
110 uint32_t idx, const void *obj_table, uint32_t n)
113 uint32_t *ring = (uint32_t *)&r[1];
114 const uint32_t *obj = (const uint32_t *)obj_table;
115 if (likely(idx + n < size)) {
116 for (i = 0; i < (n & ~0x7); i += 8, idx += 8) {
118 ring[idx + 1] = obj[i + 1];
119 ring[idx + 2] = obj[i + 2];
120 ring[idx + 3] = obj[i + 3];
121 ring[idx + 4] = obj[i + 4];
122 ring[idx + 5] = obj[i + 5];
123 ring[idx + 6] = obj[i + 6];
124 ring[idx + 7] = obj[i + 7];
128 ring[idx++] = obj[i++]; /* fallthrough */
130 ring[idx++] = obj[i++]; /* fallthrough */
132 ring[idx++] = obj[i++]; /* fallthrough */
134 ring[idx++] = obj[i++]; /* fallthrough */
136 ring[idx++] = obj[i++]; /* fallthrough */
138 ring[idx++] = obj[i++]; /* fallthrough */
140 ring[idx++] = obj[i++]; /* fallthrough */
143 for (i = 0; idx < size; i++, idx++)
145 /* Start at the beginning */
146 for (idx = 0; i < n; i++, idx++)
151 static __rte_always_inline void
152 __rte_ring_enqueue_elems_64(struct rte_ring *r, uint32_t prod_head,
153 const void *obj_table, uint32_t n)
156 const uint32_t size = r->size;
157 uint32_t idx = prod_head & r->mask;
158 uint64_t *ring = (uint64_t *)&r[1];
159 const unaligned_uint64_t *obj = (const unaligned_uint64_t *)obj_table;
160 if (likely(idx + n < size)) {
161 for (i = 0; i < (n & ~0x3); i += 4, idx += 4) {
163 ring[idx + 1] = obj[i + 1];
164 ring[idx + 2] = obj[i + 2];
165 ring[idx + 3] = obj[i + 3];
169 ring[idx++] = obj[i++]; /* fallthrough */
171 ring[idx++] = obj[i++]; /* fallthrough */
173 ring[idx++] = obj[i++];
176 for (i = 0; idx < size; i++, idx++)
178 /* Start at the beginning */
179 for (idx = 0; i < n; i++, idx++)
184 static __rte_always_inline void
185 __rte_ring_enqueue_elems_128(struct rte_ring *r, uint32_t prod_head,
186 const void *obj_table, uint32_t n)
189 const uint32_t size = r->size;
190 uint32_t idx = prod_head & r->mask;
191 rte_int128_t *ring = (rte_int128_t *)&r[1];
192 const rte_int128_t *obj = (const rte_int128_t *)obj_table;
193 if (likely(idx + n < size)) {
194 for (i = 0; i < (n & ~0x1); i += 2, idx += 2)
195 memcpy((void *)(ring + idx),
196 (const void *)(obj + i), 32);
199 memcpy((void *)(ring + idx),
200 (const void *)(obj + i), 16);
203 for (i = 0; idx < size; i++, idx++)
204 memcpy((void *)(ring + idx),
205 (const void *)(obj + i), 16);
206 /* Start at the beginning */
207 for (idx = 0; i < n; i++, idx++)
208 memcpy((void *)(ring + idx),
209 (const void *)(obj + i), 16);
213 /* the actual enqueue of elements on the ring.
214 * Placed here since identical code needed in both
215 * single and multi producer enqueue functions.
217 static __rte_always_inline void
218 __rte_ring_enqueue_elems(struct rte_ring *r, uint32_t prod_head,
219 const void *obj_table, uint32_t esize, uint32_t num)
221 /* 8B and 16B copies implemented individually to retain
222 * the current performance.
225 __rte_ring_enqueue_elems_64(r, prod_head, obj_table, num);
226 else if (esize == 16)
227 __rte_ring_enqueue_elems_128(r, prod_head, obj_table, num);
229 uint32_t idx, scale, nr_idx, nr_num, nr_size;
231 /* Normalize to uint32_t */
232 scale = esize / sizeof(uint32_t);
233 nr_num = num * scale;
234 idx = prod_head & r->mask;
235 nr_idx = idx * scale;
236 nr_size = r->size * scale;
237 __rte_ring_enqueue_elems_32(r, nr_size, nr_idx,
242 static __rte_always_inline void
243 __rte_ring_dequeue_elems_32(struct rte_ring *r, const uint32_t size,
244 uint32_t idx, void *obj_table, uint32_t n)
247 uint32_t *ring = (uint32_t *)&r[1];
248 uint32_t *obj = (uint32_t *)obj_table;
249 if (likely(idx + n < size)) {
250 for (i = 0; i < (n & ~0x7); i += 8, idx += 8) {
252 obj[i + 1] = ring[idx + 1];
253 obj[i + 2] = ring[idx + 2];
254 obj[i + 3] = ring[idx + 3];
255 obj[i + 4] = ring[idx + 4];
256 obj[i + 5] = ring[idx + 5];
257 obj[i + 6] = ring[idx + 6];
258 obj[i + 7] = ring[idx + 7];
262 obj[i++] = ring[idx++]; /* fallthrough */
264 obj[i++] = ring[idx++]; /* fallthrough */
266 obj[i++] = ring[idx++]; /* fallthrough */
268 obj[i++] = ring[idx++]; /* fallthrough */
270 obj[i++] = ring[idx++]; /* fallthrough */
272 obj[i++] = ring[idx++]; /* fallthrough */
274 obj[i++] = ring[idx++]; /* fallthrough */
277 for (i = 0; idx < size; i++, idx++)
279 /* Start at the beginning */
280 for (idx = 0; i < n; i++, idx++)
285 static __rte_always_inline void
286 __rte_ring_dequeue_elems_64(struct rte_ring *r, uint32_t prod_head,
287 void *obj_table, uint32_t n)
290 const uint32_t size = r->size;
291 uint32_t idx = prod_head & r->mask;
292 uint64_t *ring = (uint64_t *)&r[1];
293 unaligned_uint64_t *obj = (unaligned_uint64_t *)obj_table;
294 if (likely(idx + n < size)) {
295 for (i = 0; i < (n & ~0x3); i += 4, idx += 4) {
297 obj[i + 1] = ring[idx + 1];
298 obj[i + 2] = ring[idx + 2];
299 obj[i + 3] = ring[idx + 3];
303 obj[i++] = ring[idx++]; /* fallthrough */
305 obj[i++] = ring[idx++]; /* fallthrough */
307 obj[i++] = ring[idx++]; /* fallthrough */
310 for (i = 0; idx < size; i++, idx++)
312 /* Start at the beginning */
313 for (idx = 0; i < n; i++, idx++)
318 static __rte_always_inline void
319 __rte_ring_dequeue_elems_128(struct rte_ring *r, uint32_t prod_head,
320 void *obj_table, uint32_t n)
323 const uint32_t size = r->size;
324 uint32_t idx = prod_head & r->mask;
325 rte_int128_t *ring = (rte_int128_t *)&r[1];
326 rte_int128_t *obj = (rte_int128_t *)obj_table;
327 if (likely(idx + n < size)) {
328 for (i = 0; i < (n & ~0x1); i += 2, idx += 2)
329 memcpy((void *)(obj + i), (void *)(ring + idx), 32);
332 memcpy((void *)(obj + i), (void *)(ring + idx), 16);
335 for (i = 0; idx < size; i++, idx++)
336 memcpy((void *)(obj + i), (void *)(ring + idx), 16);
337 /* Start at the beginning */
338 for (idx = 0; i < n; i++, idx++)
339 memcpy((void *)(obj + i), (void *)(ring + idx), 16);
343 /* the actual dequeue of elements from the ring.
344 * Placed here since identical code needed in both
345 * single and multi producer enqueue functions.
347 static __rte_always_inline void
348 __rte_ring_dequeue_elems(struct rte_ring *r, uint32_t cons_head,
349 void *obj_table, uint32_t esize, uint32_t num)
351 /* 8B and 16B copies implemented individually to retain
352 * the current performance.
355 __rte_ring_dequeue_elems_64(r, cons_head, obj_table, num);
356 else if (esize == 16)
357 __rte_ring_dequeue_elems_128(r, cons_head, obj_table, num);
359 uint32_t idx, scale, nr_idx, nr_num, nr_size;
361 /* Normalize to uint32_t */
362 scale = esize / sizeof(uint32_t);
363 nr_num = num * scale;
364 idx = cons_head & r->mask;
365 nr_idx = idx * scale;
366 nr_size = r->size * scale;
367 __rte_ring_dequeue_elems_32(r, nr_size, nr_idx,
372 /* Between load and load. there might be cpu reorder in weak model
374 * There are 2 choices for the users
375 * 1.use rmb() memory barrier
376 * 2.use one-direction load_acquire/store_release barrier
377 * It depends on performance test results.
378 * By default, move common functions to rte_ring_generic.h
380 #ifdef RTE_USE_C11_MEM_MODEL
381 #include "rte_ring_c11_mem.h"
383 #include "rte_ring_generic.h"
387 * @internal Enqueue several objects on the ring
390 * A pointer to the ring structure.
392 * A pointer to a table of objects.
394 * The size of ring element, in bytes. It must be a multiple of 4.
395 * This must be the same value used while creating the ring. Otherwise
396 * the results are undefined.
398 * The number of objects to add in the ring from the obj_table.
400 * RTE_RING_QUEUE_FIXED: Enqueue a fixed number of items from a ring
401 * RTE_RING_QUEUE_VARIABLE: Enqueue as many items as possible from ring
403 * Indicates whether to use single producer or multi-producer head update
405 * returns the amount of space after the enqueue operation has finished
407 * Actual number of objects enqueued.
408 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
410 static __rte_always_inline unsigned int
411 __rte_ring_do_enqueue_elem(struct rte_ring *r, const void *obj_table,
412 unsigned int esize, unsigned int n,
413 enum rte_ring_queue_behavior behavior, unsigned int is_sp,
414 unsigned int *free_space)
416 uint32_t prod_head, prod_next;
417 uint32_t free_entries;
419 n = __rte_ring_move_prod_head(r, is_sp, n, behavior,
420 &prod_head, &prod_next, &free_entries);
424 __rte_ring_enqueue_elems(r, prod_head, obj_table, esize, n);
426 update_tail(&r->prod, prod_head, prod_next, is_sp, 1);
428 if (free_space != NULL)
429 *free_space = free_entries - n;
434 * @internal Dequeue several objects from the ring
437 * A pointer to the ring structure.
439 * A pointer to a table of objects.
441 * The size of ring element, in bytes. It must be a multiple of 4.
442 * This must be the same value used while creating the ring. Otherwise
443 * the results are undefined.
445 * The number of objects to pull from the ring.
447 * RTE_RING_QUEUE_FIXED: Dequeue a fixed number of items from a ring
448 * RTE_RING_QUEUE_VARIABLE: Dequeue as many items as possible from ring
450 * Indicates whether to use single consumer or multi-consumer head update
452 * returns the number of remaining ring entries after the dequeue has finished
454 * - Actual number of objects dequeued.
455 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
457 static __rte_always_inline unsigned int
458 __rte_ring_do_dequeue_elem(struct rte_ring *r, void *obj_table,
459 unsigned int esize, unsigned int n,
460 enum rte_ring_queue_behavior behavior, unsigned int is_sc,
461 unsigned int *available)
463 uint32_t cons_head, cons_next;
466 n = __rte_ring_move_cons_head(r, (int)is_sc, n, behavior,
467 &cons_head, &cons_next, &entries);
471 __rte_ring_dequeue_elems(r, cons_head, obj_table, esize, n);
473 update_tail(&r->cons, cons_head, cons_next, is_sc, 0);
476 if (available != NULL)
477 *available = entries - n;
482 * Enqueue several objects on the ring (multi-producers safe).
484 * This function uses a "compare and set" instruction to move the
485 * producer index atomically.
488 * A pointer to the ring structure.
490 * A pointer to a table of objects.
492 * The size of ring element, in bytes. It must be a multiple of 4.
493 * This must be the same value used while creating the ring. Otherwise
494 * the results are undefined.
496 * The number of objects to add in the ring from the obj_table.
498 * if non-NULL, returns the amount of space in the ring after the
499 * enqueue operation has finished.
501 * The number of objects enqueued, either 0 or n
503 static __rte_always_inline unsigned int
504 rte_ring_mp_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table,
505 unsigned int esize, unsigned int n, unsigned int *free_space)
507 return __rte_ring_do_enqueue_elem(r, obj_table, esize, n,
508 RTE_RING_QUEUE_FIXED, RTE_RING_SYNC_MT, free_space);
512 * Enqueue several objects on a ring
514 * @warning This API is NOT multi-producers safe
517 * A pointer to the ring structure.
519 * A pointer to a table of objects.
521 * The size of ring element, in bytes. It must be a multiple of 4.
522 * This must be the same value used while creating the ring. Otherwise
523 * the results are undefined.
525 * The number of objects to add in the ring from the obj_table.
527 * if non-NULL, returns the amount of space in the ring after the
528 * enqueue operation has finished.
530 * The number of objects enqueued, either 0 or n
532 static __rte_always_inline unsigned int
533 rte_ring_sp_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table,
534 unsigned int esize, unsigned int n, unsigned int *free_space)
536 return __rte_ring_do_enqueue_elem(r, obj_table, esize, n,
537 RTE_RING_QUEUE_FIXED, RTE_RING_SYNC_ST, free_space);
540 #ifdef ALLOW_EXPERIMENTAL_API
541 #include <rte_ring_hts.h>
542 #include <rte_ring_rts.h>
546 * Enqueue several objects on a ring.
548 * This function calls the multi-producer or the single-producer
549 * version depending on the default behavior that was specified at
550 * ring creation time (see flags).
553 * A pointer to the ring structure.
555 * A pointer to a table of objects.
557 * The size of ring element, in bytes. It must be a multiple of 4.
558 * This must be the same value used while creating the ring. Otherwise
559 * the results are undefined.
561 * The number of objects to add in the ring from the obj_table.
563 * if non-NULL, returns the amount of space in the ring after the
564 * enqueue operation has finished.
566 * The number of objects enqueued, either 0 or n
568 static __rte_always_inline unsigned int
569 rte_ring_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table,
570 unsigned int esize, unsigned int n, unsigned int *free_space)
572 switch (r->prod.sync_type) {
573 case RTE_RING_SYNC_MT:
574 return rte_ring_mp_enqueue_bulk_elem(r, obj_table, esize, n,
576 case RTE_RING_SYNC_ST:
577 return rte_ring_sp_enqueue_bulk_elem(r, obj_table, esize, n,
579 #ifdef ALLOW_EXPERIMENTAL_API
580 case RTE_RING_SYNC_MT_RTS:
581 return rte_ring_mp_rts_enqueue_bulk_elem(r, obj_table, esize, n,
583 case RTE_RING_SYNC_MT_HTS:
584 return rte_ring_mp_hts_enqueue_bulk_elem(r, obj_table, esize, n,
589 /* valid ring should never reach this point */
591 if (free_space != NULL)
597 * Enqueue one object on a ring (multi-producers safe).
599 * This function uses a "compare and set" instruction to move the
600 * producer index atomically.
603 * A pointer to the ring structure.
605 * A pointer to the object to be added.
607 * The size of ring element, in bytes. It must be a multiple of 4.
608 * This must be the same value used while creating the ring. Otherwise
609 * the results are undefined.
611 * - 0: Success; objects enqueued.
612 * - -ENOBUFS: Not enough room in the ring to enqueue; no object is enqueued.
614 static __rte_always_inline int
615 rte_ring_mp_enqueue_elem(struct rte_ring *r, void *obj, unsigned int esize)
617 return rte_ring_mp_enqueue_bulk_elem(r, obj, esize, 1, NULL) ? 0 :
622 * Enqueue one object on a ring
624 * @warning This API is NOT multi-producers safe
627 * A pointer to the ring structure.
629 * A pointer to the object to be added.
631 * The size of ring element, in bytes. It must be a multiple of 4.
632 * This must be the same value used while creating the ring. Otherwise
633 * the results are undefined.
635 * - 0: Success; objects enqueued.
636 * - -ENOBUFS: Not enough room in the ring to enqueue; no object is enqueued.
638 static __rte_always_inline int
639 rte_ring_sp_enqueue_elem(struct rte_ring *r, void *obj, unsigned int esize)
641 return rte_ring_sp_enqueue_bulk_elem(r, obj, esize, 1, NULL) ? 0 :
646 * Enqueue one object on a ring.
648 * This function calls the multi-producer or the single-producer
649 * version, depending on the default behaviour that was specified at
650 * ring creation time (see flags).
653 * A pointer to the ring structure.
655 * A pointer to the object to be added.
657 * The size of ring element, in bytes. It must be a multiple of 4.
658 * This must be the same value used while creating the ring. Otherwise
659 * the results are undefined.
661 * - 0: Success; objects enqueued.
662 * - -ENOBUFS: Not enough room in the ring to enqueue; no object is enqueued.
664 static __rte_always_inline int
665 rte_ring_enqueue_elem(struct rte_ring *r, void *obj, unsigned int esize)
667 return rte_ring_enqueue_bulk_elem(r, obj, esize, 1, NULL) ? 0 :
672 * Dequeue several objects from a ring (multi-consumers safe).
674 * This function uses a "compare and set" instruction to move the
675 * consumer index atomically.
678 * A pointer to the ring structure.
680 * A pointer to a table of objects that will be filled.
682 * The size of ring element, in bytes. It must be a multiple of 4.
683 * This must be the same value used while creating the ring. Otherwise
684 * the results are undefined.
686 * The number of objects to dequeue from the ring to the obj_table.
688 * If non-NULL, returns the number of remaining ring entries after the
689 * dequeue has finished.
691 * The number of objects dequeued, either 0 or n
693 static __rte_always_inline unsigned int
694 rte_ring_mc_dequeue_bulk_elem(struct rte_ring *r, void *obj_table,
695 unsigned int esize, unsigned int n, unsigned int *available)
697 return __rte_ring_do_dequeue_elem(r, obj_table, esize, n,
698 RTE_RING_QUEUE_FIXED, RTE_RING_SYNC_MT, available);
702 * Dequeue several objects from a ring (NOT multi-consumers safe).
705 * A pointer to the ring structure.
707 * A pointer to a table of objects that will be filled.
709 * The size of ring element, in bytes. It must be a multiple of 4.
710 * This must be the same value used while creating the ring. Otherwise
711 * the results are undefined.
713 * The number of objects to dequeue from the ring to the obj_table,
714 * must be strictly positive.
716 * If non-NULL, returns the number of remaining ring entries after the
717 * dequeue has finished.
719 * The number of objects dequeued, either 0 or n
721 static __rte_always_inline unsigned int
722 rte_ring_sc_dequeue_bulk_elem(struct rte_ring *r, void *obj_table,
723 unsigned int esize, unsigned int n, unsigned int *available)
725 return __rte_ring_do_dequeue_elem(r, obj_table, esize, n,
726 RTE_RING_QUEUE_FIXED, RTE_RING_SYNC_ST, available);
730 * Dequeue several objects from a ring.
732 * This function calls the multi-consumers or the single-consumer
733 * version, depending on the default behaviour that was specified at
734 * ring creation time (see flags).
737 * A pointer to the ring structure.
739 * A pointer to a table of objects that will be filled.
741 * The size of ring element, in bytes. It must be a multiple of 4.
742 * This must be the same value used while creating the ring. Otherwise
743 * the results are undefined.
745 * The number of objects to dequeue from the ring to the obj_table.
747 * If non-NULL, returns the number of remaining ring entries after the
748 * dequeue has finished.
750 * The number of objects dequeued, either 0 or n
752 static __rte_always_inline unsigned int
753 rte_ring_dequeue_bulk_elem(struct rte_ring *r, void *obj_table,
754 unsigned int esize, unsigned int n, unsigned int *available)
756 switch (r->cons.sync_type) {
757 case RTE_RING_SYNC_MT:
758 return rte_ring_mc_dequeue_bulk_elem(r, obj_table, esize, n,
760 case RTE_RING_SYNC_ST:
761 return rte_ring_sc_dequeue_bulk_elem(r, obj_table, esize, n,
763 #ifdef ALLOW_EXPERIMENTAL_API
764 case RTE_RING_SYNC_MT_RTS:
765 return rte_ring_mc_rts_dequeue_bulk_elem(r, obj_table, esize,
767 case RTE_RING_SYNC_MT_HTS:
768 return rte_ring_mc_hts_dequeue_bulk_elem(r, obj_table, esize,
773 /* valid ring should never reach this point */
775 if (available != NULL)
781 * Dequeue one object from a ring (multi-consumers safe).
783 * This function uses a "compare and set" instruction to move the
784 * consumer index atomically.
787 * A pointer to the ring structure.
789 * A pointer to the object that will be filled.
791 * The size of ring element, in bytes. It must be a multiple of 4.
792 * This must be the same value used while creating the ring. Otherwise
793 * the results are undefined.
795 * - 0: Success; objects dequeued.
796 * - -ENOENT: Not enough entries in the ring to dequeue; no object is
799 static __rte_always_inline int
800 rte_ring_mc_dequeue_elem(struct rte_ring *r, void *obj_p,
803 return rte_ring_mc_dequeue_bulk_elem(r, obj_p, esize, 1, NULL) ? 0 :
808 * Dequeue one object from a ring (NOT multi-consumers safe).
811 * A pointer to the ring structure.
813 * A pointer to the object that will be filled.
815 * The size of ring element, in bytes. It must be a multiple of 4.
816 * This must be the same value used while creating the ring. Otherwise
817 * the results are undefined.
819 * - 0: Success; objects dequeued.
820 * - -ENOENT: Not enough entries in the ring to dequeue, no object is
823 static __rte_always_inline int
824 rte_ring_sc_dequeue_elem(struct rte_ring *r, void *obj_p,
827 return rte_ring_sc_dequeue_bulk_elem(r, obj_p, esize, 1, NULL) ? 0 :
832 * Dequeue one object from a ring.
834 * This function calls the multi-consumers or the single-consumer
835 * version depending on the default behaviour that was specified at
836 * ring creation time (see flags).
839 * A pointer to the ring structure.
841 * A pointer to the object that will be filled.
843 * The size of ring element, in bytes. It must be a multiple of 4.
844 * This must be the same value used while creating the ring. Otherwise
845 * the results are undefined.
847 * - 0: Success, objects dequeued.
848 * - -ENOENT: Not enough entries in the ring to dequeue, no object is
851 static __rte_always_inline int
852 rte_ring_dequeue_elem(struct rte_ring *r, void *obj_p, unsigned int esize)
854 return rte_ring_dequeue_bulk_elem(r, obj_p, esize, 1, NULL) ? 0 :
859 * Enqueue several objects on the ring (multi-producers safe).
861 * This function uses a "compare and set" instruction to move the
862 * producer index atomically.
865 * A pointer to the ring structure.
867 * A pointer to a table of objects.
869 * The size of ring element, in bytes. It must be a multiple of 4.
870 * This must be the same value used while creating the ring. Otherwise
871 * the results are undefined.
873 * The number of objects to add in the ring from the obj_table.
875 * if non-NULL, returns the amount of space in the ring after the
876 * enqueue operation has finished.
878 * - n: Actual number of objects enqueued.
880 static __rte_always_inline unsigned int
881 rte_ring_mp_enqueue_burst_elem(struct rte_ring *r, const void *obj_table,
882 unsigned int esize, unsigned int n, unsigned int *free_space)
884 return __rte_ring_do_enqueue_elem(r, obj_table, esize, n,
885 RTE_RING_QUEUE_VARIABLE, RTE_RING_SYNC_MT, free_space);
889 * Enqueue several objects on a ring
891 * @warning This API is NOT multi-producers safe
894 * A pointer to the ring structure.
896 * A pointer to a table of objects.
898 * The size of ring element, in bytes. It must be a multiple of 4.
899 * This must be the same value used while creating the ring. Otherwise
900 * the results are undefined.
902 * The number of objects to add in the ring from the obj_table.
904 * if non-NULL, returns the amount of space in the ring after the
905 * enqueue operation has finished.
907 * - n: Actual number of objects enqueued.
909 static __rte_always_inline unsigned int
910 rte_ring_sp_enqueue_burst_elem(struct rte_ring *r, const void *obj_table,
911 unsigned int esize, unsigned int n, unsigned int *free_space)
913 return __rte_ring_do_enqueue_elem(r, obj_table, esize, n,
914 RTE_RING_QUEUE_VARIABLE, RTE_RING_SYNC_ST, free_space);
918 * Enqueue several objects on a ring.
920 * This function calls the multi-producer or the single-producer
921 * version depending on the default behavior that was specified at
922 * ring creation time (see flags).
925 * A pointer to the ring structure.
927 * A pointer to a table of objects.
929 * The size of ring element, in bytes. It must be a multiple of 4.
930 * This must be the same value used while creating the ring. Otherwise
931 * the results are undefined.
933 * The number of objects to add in the ring from the obj_table.
935 * if non-NULL, returns the amount of space in the ring after the
936 * enqueue operation has finished.
938 * - n: Actual number of objects enqueued.
940 static __rte_always_inline unsigned int
941 rte_ring_enqueue_burst_elem(struct rte_ring *r, const void *obj_table,
942 unsigned int esize, unsigned int n, unsigned int *free_space)
944 switch (r->prod.sync_type) {
945 case RTE_RING_SYNC_MT:
946 return rte_ring_mp_enqueue_burst_elem(r, obj_table, esize, n,
948 case RTE_RING_SYNC_ST:
949 return rte_ring_sp_enqueue_burst_elem(r, obj_table, esize, n,
951 #ifdef ALLOW_EXPERIMENTAL_API
952 case RTE_RING_SYNC_MT_RTS:
953 return rte_ring_mp_rts_enqueue_burst_elem(r, obj_table, esize,
955 case RTE_RING_SYNC_MT_HTS:
956 return rte_ring_mp_hts_enqueue_burst_elem(r, obj_table, esize,
961 /* valid ring should never reach this point */
963 if (free_space != NULL)
969 * Dequeue several objects from a ring (multi-consumers safe). When the request
970 * objects are more than the available objects, only dequeue the actual number
973 * This function uses a "compare and set" instruction to move the
974 * consumer index atomically.
977 * A pointer to the ring structure.
979 * A pointer to a table of objects that will be filled.
981 * The size of ring element, in bytes. It must be a multiple of 4.
982 * This must be the same value used while creating the ring. Otherwise
983 * the results are undefined.
985 * The number of objects to dequeue from the ring to the obj_table.
987 * If non-NULL, returns the number of remaining ring entries after the
988 * dequeue has finished.
990 * - n: Actual number of objects dequeued, 0 if ring is empty
992 static __rte_always_inline unsigned int
993 rte_ring_mc_dequeue_burst_elem(struct rte_ring *r, void *obj_table,
994 unsigned int esize, unsigned int n, unsigned int *available)
996 return __rte_ring_do_dequeue_elem(r, obj_table, esize, n,
997 RTE_RING_QUEUE_VARIABLE, RTE_RING_SYNC_MT, available);
1001 * Dequeue several objects from a ring (NOT multi-consumers safe).When the
1002 * request objects are more than the available objects, only dequeue the
1003 * actual number of objects
1006 * A pointer to the ring structure.
1008 * A pointer to a table of objects that will be filled.
1010 * The size of ring element, in bytes. It must be a multiple of 4.
1011 * This must be the same value used while creating the ring. Otherwise
1012 * the results are undefined.
1014 * The number of objects to dequeue from the ring to the obj_table.
1016 * If non-NULL, returns the number of remaining ring entries after the
1017 * dequeue has finished.
1019 * - n: Actual number of objects dequeued, 0 if ring is empty
1021 static __rte_always_inline unsigned int
1022 rte_ring_sc_dequeue_burst_elem(struct rte_ring *r, void *obj_table,
1023 unsigned int esize, unsigned int n, unsigned int *available)
1025 return __rte_ring_do_dequeue_elem(r, obj_table, esize, n,
1026 RTE_RING_QUEUE_VARIABLE, RTE_RING_SYNC_ST, available);
1030 * Dequeue multiple objects from a ring up to a maximum number.
1032 * This function calls the multi-consumers or the single-consumer
1033 * version, depending on the default behaviour that was specified at
1034 * ring creation time (see flags).
1037 * A pointer to the ring structure.
1039 * A pointer to a table of objects that will be filled.
1041 * The size of ring element, in bytes. It must be a multiple of 4.
1042 * This must be the same value used while creating the ring. Otherwise
1043 * the results are undefined.
1045 * The number of objects to dequeue from the ring to the obj_table.
1047 * If non-NULL, returns the number of remaining ring entries after the
1048 * dequeue has finished.
1050 * - Number of objects dequeued
1052 static __rte_always_inline unsigned int
1053 rte_ring_dequeue_burst_elem(struct rte_ring *r, void *obj_table,
1054 unsigned int esize, unsigned int n, unsigned int *available)
1056 switch (r->cons.sync_type) {
1057 case RTE_RING_SYNC_MT:
1058 return rte_ring_mc_dequeue_burst_elem(r, obj_table, esize, n,
1060 case RTE_RING_SYNC_ST:
1061 return rte_ring_sc_dequeue_burst_elem(r, obj_table, esize, n,
1063 #ifdef ALLOW_EXPERIMENTAL_API
1064 case RTE_RING_SYNC_MT_RTS:
1065 return rte_ring_mc_rts_dequeue_burst_elem(r, obj_table, esize,
1067 case RTE_RING_SYNC_MT_HTS:
1068 return rte_ring_mc_hts_dequeue_burst_elem(r, obj_table, esize,
1073 /* valid ring should never reach this point */
1075 if (available != NULL)
1080 #ifdef ALLOW_EXPERIMENTAL_API
1081 #include <rte_ring_peek.h>
1082 #include <rte_ring_peek_zc.h>
1085 #include <rte_ring.h>
1091 #endif /* _RTE_RING_ELEM_H_ */