4 * Copyright(c) 2016-2017 Intel Corporation. All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
16 * * Neither the name of Intel Corporation nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 * Generic ring structure for passing events from one core to another.
36 * Used by the software scheduler for the producer and consumer rings for
37 * each port, i.e. for passing events from worker cores to scheduler and
38 * vice-versa. Designed for single-producer, single-consumer use with two
39 * cores working on each ring.
47 #include <rte_common.h>
48 #include <rte_memory.h>
49 #include <rte_malloc.h>
51 #define QE_RING_NAMESIZE 32
54 char name[QE_RING_NAMESIZE] __rte_cache_aligned;
55 uint32_t ring_size; /* size of memory block allocated to the ring */
56 uint32_t mask; /* mask for read/write values == ring_size -1 */
57 uint32_t size; /* actual usable space in the ring */
58 volatile uint32_t write_idx __rte_cache_aligned;
59 volatile uint32_t read_idx __rte_cache_aligned;
61 struct rte_event ring[0] __rte_cache_aligned;
65 #define force_inline inline __attribute__((always_inline))
68 static inline struct qe_ring *
69 qe_ring_create(const char *name, unsigned int size, unsigned int socket_id)
71 struct qe_ring *retval;
72 const uint32_t ring_size = rte_align32pow2(size + 1);
73 size_t memsize = sizeof(*retval) +
74 (ring_size * sizeof(retval->ring[0]));
76 retval = rte_zmalloc_socket(NULL, memsize, 0, socket_id);
80 snprintf(retval->name, sizeof(retval->name), "EVDEV_RG_%s", name);
81 retval->ring_size = ring_size;
82 retval->mask = ring_size - 1;
89 qe_ring_destroy(struct qe_ring *r)
94 static force_inline unsigned int
95 qe_ring_count(const struct qe_ring *r)
97 return r->write_idx - r->read_idx;
100 static force_inline unsigned int
101 qe_ring_free_count(const struct qe_ring *r)
103 return r->size - qe_ring_count(r);
106 static force_inline unsigned int
107 qe_ring_enqueue_burst(struct qe_ring *r, const struct rte_event *qes,
108 unsigned int nb_qes, uint16_t *free_count)
110 const uint32_t size = r->size;
111 const uint32_t mask = r->mask;
112 const uint32_t read = r->read_idx;
113 uint32_t write = r->write_idx;
114 const uint32_t space = read + size - write;
120 for (i = 0; i < nb_qes; i++, write++)
121 r->ring[write & mask] = qes[i];
126 r->write_idx = write;
128 *free_count = space - nb_qes;
133 static force_inline unsigned int
134 qe_ring_enqueue_burst_with_ops(struct qe_ring *r, const struct rte_event *qes,
135 unsigned int nb_qes, uint8_t *ops)
137 const uint32_t size = r->size;
138 const uint32_t mask = r->mask;
139 const uint32_t read = r->read_idx;
140 uint32_t write = r->write_idx;
141 const uint32_t space = read + size - write;
147 for (i = 0; i < nb_qes; i++, write++) {
148 r->ring[write & mask] = qes[i];
149 r->ring[write & mask].op = ops[i];
155 r->write_idx = write;
160 static force_inline unsigned int
161 qe_ring_dequeue_burst(struct qe_ring *r, struct rte_event *qes,
164 const uint32_t mask = r->mask;
165 uint32_t read = r->read_idx;
166 const uint32_t write = r->write_idx;
167 const uint32_t items = write - read;
174 for (i = 0; i < nb_qes; i++, read++)
175 qes[i] = r->ring[read & mask];
180 r->read_idx += nb_qes;