1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (C) 2020 Marvell International Ltd.
4 #include <rte_bitmap.h>
5 #include <rte_ethdev.h>
6 #include <rte_eventdev.h>
7 #include <rte_event_eth_rx_adapter.h>
8 #include <rte_malloc.h>
11 #include "event_helper.h"
14 eh_get_enabled_cores(struct rte_bitmap *eth_core_mask)
18 RTE_LCORE_FOREACH(i) {
19 /* Check if this core is enabled in core mask*/
20 if (rte_bitmap_get(eth_core_mask, i)) {
21 /* Found enabled core */
28 static inline unsigned int
29 eh_get_next_eth_core(struct eventmode_conf *em_conf)
31 static unsigned int prev_core = -1;
32 unsigned int next_core;
35 * Make sure we have at least one eth core running, else the following
36 * logic would lead to an infinite loop.
38 if (eh_get_enabled_cores(em_conf->eth_core_mask) == 0) {
39 EH_LOG_ERR("No enabled eth core found");
43 /* Only some cores are marked as eth cores, skip others */
45 /* Get the next core */
46 next_core = rte_get_next_lcore(prev_core, 0, 1);
48 /* Check if we have reached max lcores */
49 if (next_core == RTE_MAX_LCORE)
52 /* Update prev_core */
53 prev_core = next_core;
54 } while (!(rte_bitmap_get(em_conf->eth_core_mask, next_core)));
59 static inline unsigned int
60 eh_get_next_active_core(struct eventmode_conf *em_conf, unsigned int prev_core)
62 unsigned int next_core;
64 /* Get next active core skipping cores reserved as eth cores */
66 /* Get the next core */
67 next_core = rte_get_next_lcore(prev_core, 0, 0);
69 /* Check if we have reached max lcores */
70 if (next_core == RTE_MAX_LCORE)
73 prev_core = next_core;
74 } while (rte_bitmap_get(em_conf->eth_core_mask, next_core));
80 eh_set_default_conf_eventdev(struct eventmode_conf *em_conf)
82 int lcore_count, nb_eventdev, nb_eth_dev, ret;
83 struct eventdev_params *eventdev_config;
84 struct rte_event_dev_info dev_info;
86 /* Get the number of event devices */
87 nb_eventdev = rte_event_dev_count();
88 if (nb_eventdev == 0) {
89 EH_LOG_ERR("No event devices detected");
93 if (nb_eventdev != 1) {
94 EH_LOG_ERR("Event mode does not support multiple event devices. "
95 "Please provide only one event device.");
99 /* Get the number of eth devs */
100 nb_eth_dev = rte_eth_dev_count_avail();
101 if (nb_eth_dev == 0) {
102 EH_LOG_ERR("No eth devices detected");
106 /* Get the number of lcores */
107 lcore_count = rte_lcore_count();
109 /* Read event device info */
110 ret = rte_event_dev_info_get(0, &dev_info);
112 EH_LOG_ERR("Failed to read event device info %d", ret);
116 /* Check if enough ports are available */
117 if (dev_info.max_event_ports < 2) {
118 EH_LOG_ERR("Not enough event ports available");
122 /* Get the first event dev conf */
123 eventdev_config = &(em_conf->eventdev_config[0]);
125 /* Save number of queues & ports available */
126 eventdev_config->eventdev_id = 0;
127 eventdev_config->nb_eventqueue = dev_info.max_event_queues;
128 eventdev_config->nb_eventport = dev_info.max_event_ports;
129 eventdev_config->ev_queue_mode = RTE_EVENT_QUEUE_CFG_ALL_TYPES;
131 /* Check if there are more queues than required */
132 if (eventdev_config->nb_eventqueue > nb_eth_dev + 1) {
133 /* One queue is reserved for Tx */
134 eventdev_config->nb_eventqueue = nb_eth_dev + 1;
137 /* Check if there are more ports than required */
138 if (eventdev_config->nb_eventport > lcore_count) {
139 /* One port per lcore is enough */
140 eventdev_config->nb_eventport = lcore_count;
143 /* Update the number of event devices */
144 em_conf->nb_eventdev++;
150 eh_set_default_conf_link(struct eventmode_conf *em_conf)
152 struct eventdev_params *eventdev_config;
153 struct eh_event_link_info *link;
154 unsigned int lcore_id = -1;
158 * Create a 1:1 mapping from event ports to cores. If the number
159 * of event ports is lesser than the cores, some cores won't
160 * execute worker. If there are more event ports, then some ports
166 * The event queue-port mapping is done according to the link. Since
167 * we are falling back to the default link config, enabling
168 * "all_ev_queue_to_ev_port" mode flag. This will map all queues
171 em_conf->ext_params.all_ev_queue_to_ev_port = 1;
173 /* Get first event dev conf */
174 eventdev_config = &(em_conf->eventdev_config[0]);
176 /* Loop through the ports */
177 for (i = 0; i < eventdev_config->nb_eventport; i++) {
179 /* Get next active core id */
180 lcore_id = eh_get_next_active_core(em_conf,
183 if (lcore_id == RTE_MAX_LCORE) {
184 /* Reached max cores */
188 /* Save the current combination as one link */
191 link_index = em_conf->nb_link;
193 /* Get the corresponding link */
194 link = &(em_conf->link[link_index]);
197 link->eventdev_id = eventdev_config->eventdev_id;
198 link->event_port_id = i;
199 link->lcore_id = lcore_id;
202 * Don't set eventq_id as by default all queues
203 * need to be mapped to the port, which is controlled
204 * by the operating mode.
207 /* Update number of links */
215 eh_set_default_conf_rx_adapter(struct eventmode_conf *em_conf)
217 struct rx_adapter_connection_info *conn;
218 struct eventdev_params *eventdev_config;
219 struct rx_adapter_conf *adapter;
220 bool single_ev_queue = false;
227 /* Create one adapter with eth queues mapped to event queue(s) */
229 if (em_conf->nb_eventdev == 0) {
230 EH_LOG_ERR("No event devs registered");
234 /* Get the number of eth devs */
235 nb_eth_dev = rte_eth_dev_count_avail();
237 /* Use the first event dev */
238 eventdev_config = &(em_conf->eventdev_config[0]);
240 /* Get eventdev ID */
241 eventdev_id = eventdev_config->eventdev_id;
244 /* Get adapter conf */
245 adapter = &(em_conf->rx_adapter[adapter_id]);
247 /* Set adapter conf */
248 adapter->eventdev_id = eventdev_id;
249 adapter->adapter_id = adapter_id;
250 adapter->rx_core_id = eh_get_next_eth_core(em_conf);
253 * Map all queues of eth device (port) to an event queue. If there
254 * are more event queues than eth ports then create 1:1 mapping.
255 * Otherwise map all eth ports to a single event queue.
257 if (nb_eth_dev > eventdev_config->nb_eventqueue)
258 single_ev_queue = true;
260 for (i = 0; i < nb_eth_dev; i++) {
262 /* Use only the ports enabled */
263 if ((em_conf->eth_portmask & (1 << i)) == 0)
266 /* Get the connection id */
267 conn_id = adapter->nb_connections;
269 /* Get the connection */
270 conn = &(adapter->conn[conn_id]);
272 /* Set mapping between eth ports & event queues*/
274 conn->eventq_id = single_ev_queue ? 0 : i;
276 /* Add all eth queues eth port to event queue */
277 conn->ethdev_rx_qid = -1;
279 /* Update no of connections */
280 adapter->nb_connections++;
284 /* We have setup one adapter */
285 em_conf->nb_rx_adapter = 1;
291 eh_validate_conf(struct eventmode_conf *em_conf)
296 * Check if event devs are specified. Else probe the event devices
297 * and initialize the config with all ports & queues available
299 if (em_conf->nb_eventdev == 0) {
300 ret = eh_set_default_conf_eventdev(em_conf);
306 * Check if links are specified. Else generate a default config for
307 * the event ports used.
309 if (em_conf->nb_link == 0) {
310 ret = eh_set_default_conf_link(em_conf);
316 * Check if rx adapters are specified. Else generate a default config
317 * with one rx adapter and all eth queues - event queue mapped.
319 if (em_conf->nb_rx_adapter == 0) {
320 ret = eh_set_default_conf_rx_adapter(em_conf);
329 eh_initialize_eventdev(struct eventmode_conf *em_conf)
331 struct rte_event_queue_conf eventq_conf = {0};
332 struct rte_event_dev_info evdev_default_conf;
333 struct rte_event_dev_config eventdev_conf;
334 struct eventdev_params *eventdev_config;
335 int nb_eventdev = em_conf->nb_eventdev;
336 struct eh_event_link_info *link;
337 uint8_t *queue = NULL;
343 for (i = 0; i < nb_eventdev; i++) {
345 /* Get eventdev config */
346 eventdev_config = &(em_conf->eventdev_config[i]);
348 /* Get event dev ID */
349 eventdev_id = eventdev_config->eventdev_id;
351 /* Get the number of queues */
352 nb_eventqueue = eventdev_config->nb_eventqueue;
354 /* Reset the default conf */
355 memset(&evdev_default_conf, 0,
356 sizeof(struct rte_event_dev_info));
358 /* Get default conf of eventdev */
359 ret = rte_event_dev_info_get(eventdev_id, &evdev_default_conf);
362 "Error in getting event device info[devID:%d]",
367 memset(&eventdev_conf, 0, sizeof(struct rte_event_dev_config));
368 eventdev_conf.nb_events_limit =
369 evdev_default_conf.max_num_events;
370 eventdev_conf.nb_event_queues = nb_eventqueue;
371 eventdev_conf.nb_event_ports =
372 eventdev_config->nb_eventport;
373 eventdev_conf.nb_event_queue_flows =
374 evdev_default_conf.max_event_queue_flows;
375 eventdev_conf.nb_event_port_dequeue_depth =
376 evdev_default_conf.max_event_port_dequeue_depth;
377 eventdev_conf.nb_event_port_enqueue_depth =
378 evdev_default_conf.max_event_port_enqueue_depth;
380 /* Configure event device */
381 ret = rte_event_dev_configure(eventdev_id, &eventdev_conf);
383 EH_LOG_ERR("Error in configuring event device");
387 /* Configure event queues */
388 for (j = 0; j < nb_eventqueue; j++) {
390 memset(&eventq_conf, 0,
391 sizeof(struct rte_event_queue_conf));
393 /* Per event dev queues can be ATQ or SINGLE LINK */
394 eventq_conf.event_queue_cfg =
395 eventdev_config->ev_queue_mode;
397 * All queues need to be set with sched_type as
398 * schedule type for the application stage. One queue
399 * would be reserved for the final eth tx stage. This
400 * will be an atomic queue.
402 if (j == nb_eventqueue-1) {
403 eventq_conf.schedule_type =
404 RTE_SCHED_TYPE_ATOMIC;
406 eventq_conf.schedule_type =
407 em_conf->ext_params.sched_type;
410 /* Set max atomic flows to 1024 */
411 eventq_conf.nb_atomic_flows = 1024;
412 eventq_conf.nb_atomic_order_sequences = 1024;
414 /* Setup the queue */
415 ret = rte_event_queue_setup(eventdev_id, j,
418 EH_LOG_ERR("Failed to setup event queue %d",
424 /* Configure event ports */
425 for (j = 0; j < eventdev_config->nb_eventport; j++) {
426 ret = rte_event_port_setup(eventdev_id, j, NULL);
428 EH_LOG_ERR("Failed to setup event port %d",
435 /* Make event queue - event port link */
436 for (j = 0; j < em_conf->nb_link; j++) {
439 link = &(em_conf->link[j]);
441 /* Get event dev ID */
442 eventdev_id = link->eventdev_id;
445 * If "all_ev_queue_to_ev_port" params flag is selected, all
446 * queues need to be mapped to the port.
448 if (em_conf->ext_params.all_ev_queue_to_ev_port)
451 queue = &(link->eventq_id);
453 /* Link queue to port */
454 ret = rte_event_port_link(eventdev_id, link->event_port_id,
457 EH_LOG_ERR("Failed to link event port %d", ret);
462 /* Start event devices */
463 for (i = 0; i < nb_eventdev; i++) {
465 /* Get eventdev config */
466 eventdev_config = &(em_conf->eventdev_config[i]);
468 ret = rte_event_dev_start(eventdev_config->eventdev_id);
470 EH_LOG_ERR("Failed to start event device %d, %d",
479 eh_rx_adapter_configure(struct eventmode_conf *em_conf,
480 struct rx_adapter_conf *adapter)
482 struct rte_event_eth_rx_adapter_queue_conf queue_conf = {0};
483 struct rte_event_dev_info evdev_default_conf = {0};
484 struct rte_event_port_conf port_conf = {0};
485 struct rx_adapter_connection_info *conn;
491 /* Get event dev ID */
492 eventdev_id = adapter->eventdev_id;
494 /* Get default configuration of event dev */
495 ret = rte_event_dev_info_get(eventdev_id, &evdev_default_conf);
497 EH_LOG_ERR("Failed to get event dev info %d", ret);
501 /* Setup port conf */
502 port_conf.new_event_threshold = 1200;
503 port_conf.dequeue_depth =
504 evdev_default_conf.max_event_port_dequeue_depth;
505 port_conf.enqueue_depth =
506 evdev_default_conf.max_event_port_enqueue_depth;
508 /* Create Rx adapter */
509 ret = rte_event_eth_rx_adapter_create(adapter->adapter_id,
510 adapter->eventdev_id, &port_conf);
512 EH_LOG_ERR("Failed to create rx adapter %d", ret);
516 /* Setup various connections in the adapter */
517 for (j = 0; j < adapter->nb_connections; j++) {
519 conn = &(adapter->conn[j]);
521 /* Setup queue conf */
522 queue_conf.ev.queue_id = conn->eventq_id;
523 queue_conf.ev.sched_type = em_conf->ext_params.sched_type;
524 queue_conf.ev.event_type = RTE_EVENT_TYPE_ETHDEV;
526 /* Add queue to the adapter */
527 ret = rte_event_eth_rx_adapter_queue_add(adapter->adapter_id,
528 conn->ethdev_id, conn->ethdev_rx_qid,
531 EH_LOG_ERR("Failed to add eth queue to rx adapter %d",
537 /* Get the service ID used by rx adapter */
538 ret = rte_event_eth_rx_adapter_service_id_get(adapter->adapter_id,
540 if (ret != -ESRCH && ret < 0) {
541 EH_LOG_ERR("Failed to get service id used by rx adapter %d",
546 rte_service_set_runstate_mapped_check(service_id, 0);
549 ret = rte_event_eth_rx_adapter_start(adapter->adapter_id);
551 EH_LOG_ERR("Failed to start rx adapter %d", ret);
559 eh_initialize_rx_adapter(struct eventmode_conf *em_conf)
561 struct rx_adapter_conf *adapter;
564 /* Configure rx adapters */
565 for (i = 0; i < em_conf->nb_rx_adapter; i++) {
566 adapter = &(em_conf->rx_adapter[i]);
567 ret = eh_rx_adapter_configure(em_conf, adapter);
569 EH_LOG_ERR("Failed to configure rx adapter %d", ret);
577 eh_devs_init(struct eh_conf *conf)
579 struct eventmode_conf *em_conf;
584 EH_LOG_ERR("Invalid event helper configuration");
588 if (conf->mode != EH_PKT_TRANSFER_MODE_EVENT)
591 if (conf->mode_params == NULL) {
592 EH_LOG_ERR("Invalid event mode parameters");
596 /* Get eventmode conf */
597 em_conf = conf->mode_params;
599 /* Eventmode conf would need eth portmask */
600 em_conf->eth_portmask = conf->eth_portmask;
602 /* Validate the requested config */
603 ret = eh_validate_conf(em_conf);
605 EH_LOG_ERR("Failed to validate the requested config %d", ret);
609 /* Stop eth devices before setting up adapter */
610 RTE_ETH_FOREACH_DEV(port_id) {
612 /* Use only the ports enabled */
613 if ((conf->eth_portmask & (1 << port_id)) == 0)
616 rte_eth_dev_stop(port_id);
620 ret = eh_initialize_eventdev(em_conf);
622 EH_LOG_ERR("Failed to initialize event dev %d", ret);
626 /* Setup Rx adapter */
627 ret = eh_initialize_rx_adapter(em_conf);
629 EH_LOG_ERR("Failed to initialize rx adapter %d", ret);
633 /* Start eth devices after setting up adapter */
634 RTE_ETH_FOREACH_DEV(port_id) {
636 /* Use only the ports enabled */
637 if ((conf->eth_portmask & (1 << port_id)) == 0)
640 ret = rte_eth_dev_start(port_id);
642 EH_LOG_ERR("Failed to start eth dev %d, %d",
652 eh_devs_uninit(struct eh_conf *conf)
654 struct eventmode_conf *em_conf;
659 EH_LOG_ERR("Invalid event helper configuration");
663 if (conf->mode != EH_PKT_TRANSFER_MODE_EVENT)
666 if (conf->mode_params == NULL) {
667 EH_LOG_ERR("Invalid event mode parameters");
671 /* Get eventmode conf */
672 em_conf = conf->mode_params;
674 /* Stop and release rx adapters */
675 for (i = 0; i < em_conf->nb_rx_adapter; i++) {
677 id = em_conf->rx_adapter[i].adapter_id;
678 ret = rte_event_eth_rx_adapter_stop(id);
680 EH_LOG_ERR("Failed to stop rx adapter %d", ret);
684 for (j = 0; j < em_conf->rx_adapter[i].nb_connections; j++) {
686 ret = rte_event_eth_rx_adapter_queue_del(id,
687 em_conf->rx_adapter[i].conn[j].ethdev_id, -1);
690 "Failed to remove rx adapter queues %d",
696 ret = rte_event_eth_rx_adapter_free(id);
698 EH_LOG_ERR("Failed to free rx adapter %d", ret);
703 /* Stop and release event devices */
704 for (i = 0; i < em_conf->nb_eventdev; i++) {
706 id = em_conf->eventdev_config[i].eventdev_id;
707 rte_event_dev_stop(id);
709 ret = rte_event_dev_close(id);
711 EH_LOG_ERR("Failed to close event dev %d, %d", id, ret);