1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
8 #include <rte_atomic.h>
9 #include <rte_cycles.h>
10 #include <rte_ethdev.h>
11 #include <rte_eventdev.h>
12 #include <rte_event_eth_rx_adapter.h>
13 #include <rte_event_eth_tx_adapter.h>
14 #include <rte_lcore.h>
15 #include <rte_malloc.h>
16 #include <rte_spinlock.h>
18 #include "l2fwd_event.h"
20 #define L2FWD_EVENT_SINGLE 0x1
21 #define L2FWD_EVENT_BURST 0x2
22 #define L2FWD_EVENT_TX_DIRECT 0x4
23 #define L2FWD_EVENT_TX_ENQ 0x8
24 #define L2FWD_EVENT_UPDT_MAC 0x10
27 l2fwd_event_service_enable(uint32_t service_id)
29 uint8_t min_service_count = UINT8_MAX;
30 uint32_t slcore_array[RTE_MAX_LCORE];
31 unsigned int slcore = 0;
32 uint8_t service_count;
35 if (!rte_service_lcore_count())
38 slcore_count = rte_service_lcore_list(slcore_array, RTE_MAX_LCORE);
41 /* Get the core which has least number of services running. */
42 while (slcore_count--) {
43 /* Reset default mapping */
44 if (rte_service_map_lcore_set(service_id,
45 slcore_array[slcore_count], 0) != 0)
47 service_count = rte_service_lcore_count_services(
48 slcore_array[slcore_count]);
49 if (service_count < min_service_count) {
50 slcore = slcore_array[slcore_count];
51 min_service_count = service_count;
54 if (rte_service_map_lcore_set(service_id, slcore, 1) != 0)
56 rte_service_lcore_start(slcore);
62 l2fwd_event_service_setup(struct l2fwd_resources *rsrc)
64 struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
65 struct rte_event_dev_info evdev_info;
66 uint32_t service_id, caps;
69 /* Running eventdev scheduler service on service core. 8< */
70 rte_event_dev_info_get(evt_rsrc->event_d_id, &evdev_info);
71 if (!(evdev_info.event_dev_cap & RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED)) {
72 ret = rte_event_dev_service_id_get(evt_rsrc->event_d_id,
74 if (ret != -ESRCH && ret != 0)
75 rte_panic("Error in starting eventdev service\n");
76 l2fwd_event_service_enable(service_id);
78 /* >8 End of running eventdev scheduler service on service core. */
80 /* Gets service ID for RX/TX adapters. 8< */
81 for (i = 0; i < evt_rsrc->rx_adptr.nb_rx_adptr; i++) {
82 ret = rte_event_eth_rx_adapter_caps_get(evt_rsrc->event_d_id,
83 evt_rsrc->rx_adptr.rx_adptr[i], &caps);
85 rte_panic("Failed to get Rx adapter[%d] caps\n",
86 evt_rsrc->rx_adptr.rx_adptr[i]);
87 ret = rte_event_eth_rx_adapter_service_id_get(
90 if (ret != -ESRCH && ret != 0)
91 rte_panic("Error in starting Rx adapter[%d] service\n",
92 evt_rsrc->rx_adptr.rx_adptr[i]);
93 l2fwd_event_service_enable(service_id);
96 for (i = 0; i < evt_rsrc->tx_adptr.nb_tx_adptr; i++) {
97 ret = rte_event_eth_tx_adapter_caps_get(evt_rsrc->event_d_id,
98 evt_rsrc->tx_adptr.tx_adptr[i], &caps);
100 rte_panic("Failed to get Rx adapter[%d] caps\n",
101 evt_rsrc->tx_adptr.tx_adptr[i]);
102 ret = rte_event_eth_tx_adapter_service_id_get(
103 evt_rsrc->event_d_id,
105 if (ret != -ESRCH && ret != 0)
106 rte_panic("Error in starting Rx adapter[%d] service\n",
107 evt_rsrc->tx_adptr.tx_adptr[i]);
108 l2fwd_event_service_enable(service_id);
110 /* >8 End of get service ID for RX/TX adapters. */
114 l2fwd_event_capability_setup(struct l2fwd_event_resources *evt_rsrc)
120 RTE_ETH_FOREACH_DEV(i) {
121 ret = rte_event_eth_tx_adapter_caps_get(0, i, &caps);
123 rte_panic("Invalid capability for Tx adptr port %d\n",
126 evt_rsrc->tx_mode_q |= !(caps &
127 RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT);
130 if (evt_rsrc->tx_mode_q)
131 l2fwd_event_set_generic_ops(&evt_rsrc->ops);
133 l2fwd_event_set_internal_port_ops(&evt_rsrc->ops);
136 static __rte_noinline int
137 l2fwd_get_free_event_port(struct l2fwd_event_resources *evt_rsrc)
142 rte_spinlock_lock(&evt_rsrc->evp.lock);
143 if (index >= evt_rsrc->evp.nb_ports) {
144 printf("No free event port is available\n");
148 port_id = evt_rsrc->evp.event_p_id[index];
150 rte_spinlock_unlock(&evt_rsrc->evp.lock);
155 static __rte_always_inline void
156 l2fwd_event_fwd(struct l2fwd_resources *rsrc, struct rte_event *ev,
157 const uint8_t tx_q_id, const uint64_t timer_period,
158 const uint32_t flags)
160 struct rte_mbuf *mbuf = ev->mbuf;
163 rte_prefetch0(rte_pktmbuf_mtod(mbuf, void *));
164 dst_port = rsrc->dst_ports[mbuf->port];
166 if (timer_period > 0)
167 __atomic_fetch_add(&rsrc->port_stats[mbuf->port].rx,
168 1, __ATOMIC_RELAXED);
169 mbuf->port = dst_port;
171 if (flags & L2FWD_EVENT_UPDT_MAC)
172 l2fwd_mac_updating(mbuf, dst_port, &rsrc->eth_addr[dst_port]);
174 if (flags & L2FWD_EVENT_TX_ENQ) {
175 ev->queue_id = tx_q_id;
176 ev->op = RTE_EVENT_OP_FORWARD;
179 if (flags & L2FWD_EVENT_TX_DIRECT)
180 rte_event_eth_tx_adapter_txq_set(mbuf, 0);
182 if (timer_period > 0)
183 __atomic_fetch_add(&rsrc->port_stats[mbuf->port].tx,
184 1, __ATOMIC_RELAXED);
187 static __rte_always_inline void
188 l2fwd_event_loop_single(struct l2fwd_resources *rsrc,
189 const uint32_t flags)
191 struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
192 const int port_id = l2fwd_get_free_event_port(evt_rsrc);
193 const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
194 evt_rsrc->evq.nb_queues - 1];
195 const uint64_t timer_period = rsrc->timer_period;
196 const uint8_t event_d_id = evt_rsrc->event_d_id;
202 printf("%s(): entering eventdev main loop on lcore %u\n", __func__,
205 while (!rsrc->force_quit) {
206 /* Read packet from eventdev */
207 if (!rte_event_dequeue_burst(event_d_id, port_id, &ev, 1, 0))
210 l2fwd_event_fwd(rsrc, &ev, tx_q_id, timer_period, flags);
212 if (flags & L2FWD_EVENT_TX_ENQ) {
213 while (rte_event_enqueue_burst(event_d_id, port_id,
219 if (flags & L2FWD_EVENT_TX_DIRECT) {
220 while (!rte_event_eth_tx_adapter_enqueue(event_d_id,
229 static __rte_always_inline void
230 l2fwd_event_loop_burst(struct l2fwd_resources *rsrc,
231 const uint32_t flags)
233 struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
234 const int port_id = l2fwd_get_free_event_port(evt_rsrc);
235 const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
236 evt_rsrc->evq.nb_queues - 1];
237 const uint64_t timer_period = rsrc->timer_period;
238 const uint8_t event_d_id = evt_rsrc->event_d_id;
239 const uint8_t deq_len = evt_rsrc->deq_depth;
240 struct rte_event ev[MAX_PKT_BURST];
241 uint16_t nb_rx, nb_tx;
247 printf("%s(): entering eventdev main loop on lcore %u\n", __func__,
250 while (!rsrc->force_quit) {
251 /* Read packet from eventdev. 8< */
252 nb_rx = rte_event_dequeue_burst(event_d_id, port_id, ev,
257 for (i = 0; i < nb_rx; i++) {
258 l2fwd_event_fwd(rsrc, &ev[i], tx_q_id, timer_period,
261 /* >8 End of reading packets from eventdev. */
263 if (flags & L2FWD_EVENT_TX_ENQ) {
264 /* Forwarding to destination ports. 8< */
265 nb_tx = rte_event_enqueue_burst(event_d_id, port_id,
267 while (nb_tx < nb_rx && !rsrc->force_quit)
268 nb_tx += rte_event_enqueue_burst(event_d_id,
271 /* >8 End of forwarding to destination ports. */
274 if (flags & L2FWD_EVENT_TX_DIRECT) {
275 nb_tx = rte_event_eth_tx_adapter_enqueue(event_d_id,
278 while (nb_tx < nb_rx && !rsrc->force_quit)
279 nb_tx += rte_event_eth_tx_adapter_enqueue(
281 ev + nb_tx, nb_rx - nb_tx, 0);
286 static __rte_always_inline void
287 l2fwd_event_loop(struct l2fwd_resources *rsrc,
288 const uint32_t flags)
290 if (flags & L2FWD_EVENT_SINGLE)
291 l2fwd_event_loop_single(rsrc, flags);
292 if (flags & L2FWD_EVENT_BURST)
293 l2fwd_event_loop_burst(rsrc, flags);
296 static void __rte_noinline
297 l2fwd_event_main_loop_tx_d(struct l2fwd_resources *rsrc)
299 l2fwd_event_loop(rsrc,
300 L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_SINGLE);
303 static void __rte_noinline
304 l2fwd_event_main_loop_tx_d_brst(struct l2fwd_resources *rsrc)
306 l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_BURST);
309 static void __rte_noinline
310 l2fwd_event_main_loop_tx_q(struct l2fwd_resources *rsrc)
312 l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_SINGLE);
315 static void __rte_noinline
316 l2fwd_event_main_loop_tx_q_brst(struct l2fwd_resources *rsrc)
318 l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_BURST);
321 static void __rte_noinline
322 l2fwd_event_main_loop_tx_d_mac(struct l2fwd_resources *rsrc)
324 l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
325 L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_SINGLE);
328 static void __rte_noinline
329 l2fwd_event_main_loop_tx_d_brst_mac(struct l2fwd_resources *rsrc)
331 l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
332 L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_BURST);
335 static void __rte_noinline
336 l2fwd_event_main_loop_tx_q_mac(struct l2fwd_resources *rsrc)
338 l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
339 L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_SINGLE);
342 static void __rte_noinline
343 l2fwd_event_main_loop_tx_q_brst_mac(struct l2fwd_resources *rsrc)
345 l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
346 L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_BURST);
350 l2fwd_event_resource_setup(struct l2fwd_resources *rsrc)
352 /* [MAC_UPDT][TX_MODE][BURST] */
353 const event_loop_cb event_loop[2][2][2] = {
354 [0][0][0] = l2fwd_event_main_loop_tx_d,
355 [0][0][1] = l2fwd_event_main_loop_tx_d_brst,
356 [0][1][0] = l2fwd_event_main_loop_tx_q,
357 [0][1][1] = l2fwd_event_main_loop_tx_q_brst,
358 [1][0][0] = l2fwd_event_main_loop_tx_d_mac,
359 [1][0][1] = l2fwd_event_main_loop_tx_d_brst_mac,
360 [1][1][0] = l2fwd_event_main_loop_tx_q_mac,
361 [1][1][1] = l2fwd_event_main_loop_tx_q_brst_mac,
363 struct l2fwd_event_resources *evt_rsrc;
364 uint32_t event_queue_cfg;
367 if (!rte_event_dev_count())
368 rte_panic("No Eventdev found\n");
370 evt_rsrc = rte_zmalloc("l2fwd_event",
371 sizeof(struct l2fwd_event_resources), 0);
372 if (evt_rsrc == NULL)
373 rte_panic("Failed to allocate memory\n");
375 rsrc->evt_rsrc = evt_rsrc;
377 /* Setup eventdev capability callbacks */
378 l2fwd_event_capability_setup(evt_rsrc);
380 /* Event device configuration */
381 event_queue_cfg = evt_rsrc->ops.event_device_setup(rsrc);
383 /* Event queue configuration */
384 evt_rsrc->ops.event_queue_setup(rsrc, event_queue_cfg);
386 /* Event port configuration */
387 evt_rsrc->ops.event_port_setup(rsrc);
389 /* Rx/Tx adapters configuration */
390 evt_rsrc->ops.adapter_setup(rsrc);
392 /* Start event device */
393 ret = rte_event_dev_start(evt_rsrc->event_d_id);
395 rte_panic("Error in starting eventdev\n");
397 evt_rsrc->ops.l2fwd_event_loop = event_loop
399 [evt_rsrc->tx_mode_q]
400 [evt_rsrc->has_burst];