1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2017 Intel Corporation
8 #include <rte_bus_vdev.h>
9 #include <rte_kvargs.h>
11 #include <rte_errno.h>
12 #include <rte_event_ring.h>
13 #include <rte_service_component.h>
17 #include "event_ring.h"
19 #define EVENTDEV_NAME_SW_PMD event_sw
20 #define NUMA_NODE_ARG "numa_node"
21 #define SCHED_QUANTA_ARG "sched_quanta"
22 #define CREDIT_QUANTA_ARG "credit_quanta"
23 #define MIN_BURST_SIZE_ARG "min_burst"
24 #define DEQ_BURST_SIZE_ARG "deq_burst"
25 #define REFIL_ONCE_ARG "refill_once"
28 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info);
31 sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[],
32 const uint8_t priorities[], uint16_t num)
34 struct sw_port *p = port;
35 struct sw_evdev *sw = sw_pmd_priv(dev);
38 RTE_SET_USED(priorities);
39 for (i = 0; i < num; i++) {
40 struct sw_qid *q = &sw->qids[queues[i]];
43 /* check for qid map overflow */
44 if (q->cq_num_mapped_cqs >= RTE_DIM(q->cq_map)) {
49 if (p->is_directed && p->num_qids_mapped > 0) {
54 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
55 if (q->cq_map[j] == p->id)
59 /* check if port is already linked */
60 if (j < q->cq_num_mapped_cqs)
63 if (q->type == SW_SCHED_TYPE_DIRECT) {
64 /* check directed qids only map to one port */
65 if (p->num_qids_mapped > 0) {
69 /* check port only takes a directed flow */
76 p->num_qids_mapped = 1;
77 } else if (q->type == RTE_SCHED_TYPE_ORDERED) {
78 p->num_ordered_qids++;
80 } else if (q->type == RTE_SCHED_TYPE_ATOMIC ||
81 q->type == RTE_SCHED_TYPE_PARALLEL) {
85 q->cq_map[q->cq_num_mapped_cqs] = p->id;
87 q->cq_num_mapped_cqs++;
93 sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[],
96 struct sw_port *p = port;
97 struct sw_evdev *sw = sw_pmd_priv(dev);
101 for (i = 0; i < nb_unlinks; i++) {
102 struct sw_qid *q = &sw->qids[queues[i]];
103 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
104 if (q->cq_map[j] == p->id) {
106 q->cq_map[q->cq_num_mapped_cqs - 1];
108 q->cq_num_mapped_cqs--;
111 p->num_qids_mapped--;
113 if (q->type == RTE_SCHED_TYPE_ORDERED)
114 p->num_ordered_qids--;
121 p->unlinks_in_progress += unlinked;
128 sw_port_unlinks_in_progress(struct rte_eventdev *dev, void *port)
131 struct sw_port *p = port;
132 return p->unlinks_in_progress;
136 sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
137 const struct rte_event_port_conf *conf)
139 struct sw_evdev *sw = sw_pmd_priv(dev);
140 struct sw_port *p = &sw->ports[port_id];
141 char buf[RTE_RING_NAMESIZE];
144 struct rte_event_dev_info info;
145 sw_info_get(dev, &info);
147 /* detect re-configuring and return credits to instance if needed */
148 if (p->initialized) {
149 /* taking credits from pool is done one quanta at a time, and
150 * credits may be spend (counted in p->inflights) or still
151 * available in the port (p->inflight_credits). We must return
152 * the sum to no leak credits
154 int possible_inflights = p->inflight_credits + p->inflights;
155 rte_atomic32_sub(&sw->inflights, possible_inflights);
158 *p = (struct sw_port){0}; /* zero entire structure */
162 /* check to see if rings exists - port_setup() can be called multiple
163 * times legally (assuming device is stopped). If ring exists, free it
164 * to so it gets re-created with the correct size
166 snprintf(buf, sizeof(buf), "sw%d_p%u_%s", dev->data->dev_id,
167 port_id, "rx_worker_ring");
168 struct rte_event_ring *existing_ring = rte_event_ring_lookup(buf);
170 rte_event_ring_free(existing_ring);
172 p->rx_worker_ring = rte_event_ring_create(buf, MAX_SW_PROD_Q_DEPTH,
173 dev->data->socket_id,
174 RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
175 if (p->rx_worker_ring == NULL) {
176 SW_LOG_ERR("Error creating RX worker ring for port %d\n",
181 p->inflight_max = conf->new_event_threshold;
182 p->implicit_release = !(conf->event_port_cfg &
183 RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
185 /* check if ring exists, same as rx_worker above */
186 snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id,
187 port_id, "cq_worker_ring");
188 existing_ring = rte_event_ring_lookup(buf);
190 rte_event_ring_free(existing_ring);
192 p->cq_worker_ring = rte_event_ring_create(buf, conf->dequeue_depth,
193 dev->data->socket_id,
194 RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
195 if (p->cq_worker_ring == NULL) {
196 rte_event_ring_free(p->rx_worker_ring);
197 SW_LOG_ERR("Error creating CQ worker ring for port %d\n",
201 sw->cq_ring_space[port_id] = conf->dequeue_depth;
203 /* set hist list contents to empty */
204 for (i = 0; i < SW_PORT_HIST_LIST; i++) {
205 p->hist_list[i].fid = -1;
206 p->hist_list[i].qid = -1;
208 dev->data->ports[port_id] = p;
216 sw_port_release(void *port)
218 struct sw_port *p = (void *)port;
222 rte_event_ring_free(p->rx_worker_ring);
223 rte_event_ring_free(p->cq_worker_ring);
224 memset(p, 0, sizeof(*p));
228 qid_init(struct sw_evdev *sw, unsigned int idx, int type,
229 const struct rte_event_queue_conf *queue_conf)
232 int dev_id = sw->data->dev_id;
233 int socket_id = sw->data->socket_id;
234 char buf[IQ_ROB_NAMESIZE];
235 struct sw_qid *qid = &sw->qids[idx];
237 /* Initialize the FID structures to no pinning (-1), and zero packets */
238 const struct sw_fid_t fid = {.cq = -1, .pcount = 0};
239 for (i = 0; i < RTE_DIM(qid->fids); i++)
244 qid->priority = queue_conf->priority;
246 if (qid->type == RTE_SCHED_TYPE_ORDERED) {
247 uint32_t window_size;
249 /* rte_ring and window_size_mask require require window_size to
252 window_size = rte_align32pow2(
253 queue_conf->nb_atomic_order_sequences);
255 qid->window_size = window_size - 1;
259 "invalid reorder_window_size for ordered queue\n"
264 snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i);
265 qid->reorder_buffer = rte_zmalloc_socket(buf,
266 window_size * sizeof(qid->reorder_buffer[0]),
268 if (!qid->reorder_buffer) {
269 SW_LOG_DBG("reorder_buffer malloc failed\n");
273 memset(&qid->reorder_buffer[0],
275 window_size * sizeof(qid->reorder_buffer[0]));
277 qid->reorder_buffer_freelist = rob_ring_create(window_size,
279 if (!qid->reorder_buffer_freelist) {
280 SW_LOG_DBG("freelist ring create failed");
284 /* Populate the freelist with reorder buffer entries. Enqueue
285 * 'window_size - 1' entries because the rte_ring holds only
288 for (i = 0; i < window_size - 1; i++) {
289 if (rob_ring_enqueue(qid->reorder_buffer_freelist,
290 &qid->reorder_buffer[i]) != 1)
294 qid->reorder_buffer_index = 0;
298 qid->initialized = 1;
303 if (qid->reorder_buffer) {
304 rte_free(qid->reorder_buffer);
305 qid->reorder_buffer = NULL;
308 if (qid->reorder_buffer_freelist) {
309 rob_ring_free(qid->reorder_buffer_freelist);
310 qid->reorder_buffer_freelist = NULL;
317 sw_queue_release(struct rte_eventdev *dev, uint8_t id)
319 struct sw_evdev *sw = sw_pmd_priv(dev);
320 struct sw_qid *qid = &sw->qids[id];
322 if (qid->type == RTE_SCHED_TYPE_ORDERED) {
323 rte_free(qid->reorder_buffer);
324 rob_ring_free(qid->reorder_buffer_freelist);
326 memset(qid, 0, sizeof(*qid));
330 sw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
331 const struct rte_event_queue_conf *conf)
335 type = conf->schedule_type;
337 if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) {
338 type = SW_SCHED_TYPE_DIRECT;
339 } else if (RTE_EVENT_QUEUE_CFG_ALL_TYPES
340 & conf->event_queue_cfg) {
341 SW_LOG_ERR("QUEUE_CFG_ALL_TYPES not supported\n");
345 struct sw_evdev *sw = sw_pmd_priv(dev);
347 if (sw->qids[queue_id].initialized)
348 sw_queue_release(dev, queue_id);
350 return qid_init(sw, queue_id, type, conf);
354 sw_init_qid_iqs(struct sw_evdev *sw)
358 /* Initialize the IQ memory of all configured qids */
359 for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) {
360 struct sw_qid *qid = &sw->qids[i];
362 if (!qid->initialized)
365 for (j = 0; j < SW_IQS_MAX; j++)
366 iq_init(sw, &qid->iq[j]);
371 sw_qids_empty(struct sw_evdev *sw)
375 for (i = 0; i < sw->qid_count; i++) {
376 for (j = 0; j < SW_IQS_MAX; j++) {
377 if (iq_count(&sw->qids[i].iq[j]))
386 sw_ports_empty(struct sw_evdev *sw)
390 for (i = 0; i < sw->port_count; i++) {
391 if ((rte_event_ring_count(sw->ports[i].rx_worker_ring)) ||
392 rte_event_ring_count(sw->ports[i].cq_worker_ring))
400 sw_drain_ports(struct rte_eventdev *dev)
402 struct sw_evdev *sw = sw_pmd_priv(dev);
403 eventdev_stop_flush_t flush;
408 flush = dev->dev_ops->dev_stop_flush;
409 dev_id = dev->data->dev_id;
410 arg = dev->data->dev_stop_flush_arg;
412 for (i = 0; i < sw->port_count; i++) {
415 while (rte_event_dequeue_burst(dev_id, i, &ev, 1, 0)) {
417 flush(dev_id, ev, arg);
419 ev.op = RTE_EVENT_OP_RELEASE;
420 rte_event_enqueue_burst(dev_id, i, &ev, 1);
426 sw_drain_queue(struct rte_eventdev *dev, struct sw_iq *iq)
428 struct sw_evdev *sw = sw_pmd_priv(dev);
429 eventdev_stop_flush_t flush;
433 flush = dev->dev_ops->dev_stop_flush;
434 dev_id = dev->data->dev_id;
435 arg = dev->data->dev_stop_flush_arg;
437 while (iq_count(iq) > 0) {
440 iq_dequeue_burst(sw, iq, &ev, 1);
443 flush(dev_id, ev, arg);
448 sw_drain_queues(struct rte_eventdev *dev)
450 struct sw_evdev *sw = sw_pmd_priv(dev);
453 for (i = 0; i < sw->qid_count; i++) {
454 for (j = 0; j < SW_IQS_MAX; j++)
455 sw_drain_queue(dev, &sw->qids[i].iq[j]);
460 sw_clean_qid_iqs(struct rte_eventdev *dev)
462 struct sw_evdev *sw = sw_pmd_priv(dev);
465 /* Release the IQ memory of all configured qids */
466 for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) {
467 struct sw_qid *qid = &sw->qids[i];
469 for (j = 0; j < SW_IQS_MAX; j++) {
470 if (!qid->iq[j].head)
472 iq_free_chunk_list(sw, qid->iq[j].head);
473 qid->iq[j].head = NULL;
479 sw_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id,
480 struct rte_event_queue_conf *conf)
483 RTE_SET_USED(queue_id);
485 static const struct rte_event_queue_conf default_conf = {
486 .nb_atomic_flows = 4096,
487 .nb_atomic_order_sequences = 1,
488 .schedule_type = RTE_SCHED_TYPE_ATOMIC,
489 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
492 *conf = default_conf;
496 sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
497 struct rte_event_port_conf *port_conf)
500 RTE_SET_USED(port_id);
502 port_conf->new_event_threshold = 1024;
503 port_conf->dequeue_depth = 16;
504 port_conf->enqueue_depth = 16;
505 port_conf->event_port_cfg = 0;
509 sw_dev_configure(const struct rte_eventdev *dev)
511 struct sw_evdev *sw = sw_pmd_priv(dev);
512 const struct rte_eventdev_data *data = dev->data;
513 const struct rte_event_dev_config *conf = &data->dev_conf;
516 sw->qid_count = conf->nb_event_queues;
517 sw->port_count = conf->nb_event_ports;
518 sw->nb_events_limit = conf->nb_events_limit;
519 rte_atomic32_set(&sw->inflights, 0);
521 /* Number of chunks sized for worst-case spread of events across IQs */
522 num_chunks = ((SW_INFLIGHT_EVENTS_TOTAL/SW_EVS_PER_Q_CHUNK)+1) +
523 sw->qid_count*SW_IQS_MAX*2;
525 /* If this is a reconfiguration, free the previous IQ allocation. All
526 * IQ chunk references were cleaned out of the QIDs in sw_stop(), and
527 * will be reinitialized in sw_start().
529 rte_free(sw->chunks);
531 sw->chunks = rte_malloc_socket(NULL,
532 sizeof(struct sw_queue_chunk) *
535 sw->data->socket_id);
539 sw->chunk_list_head = NULL;
540 for (i = 0; i < num_chunks; i++)
541 iq_free_chunk(sw, &sw->chunks[i]);
543 if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
552 sw_eth_rx_adapter_caps_get(const struct rte_eventdev *dev,
553 const struct rte_eth_dev *eth_dev,
557 RTE_SET_USED(eth_dev);
558 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP;
563 sw_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags,
565 const struct event_timer_adapter_ops **ops)
571 /* Use default SW ops */
578 sw_crypto_adapter_caps_get(const struct rte_eventdev *dev,
579 const struct rte_cryptodev *cdev,
584 *caps = RTE_EVENT_CRYPTO_ADAPTER_SW_CAP;
589 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info)
593 static const struct rte_event_dev_info evdev_sw_info = {
594 .driver_name = SW_PMD_NAME,
595 .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
596 .max_event_queue_flows = SW_QID_NUM_FIDS,
597 .max_event_queue_priority_levels = SW_Q_PRIORITY_MAX,
598 .max_event_priority_levels = SW_IQS_MAX,
599 .max_event_ports = SW_PORTS_MAX,
600 .max_event_port_dequeue_depth = MAX_SW_CONS_Q_DEPTH,
601 .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH,
602 .max_num_events = SW_INFLIGHT_EVENTS_TOTAL,
604 RTE_EVENT_DEV_CAP_QUEUE_QOS |
605 RTE_EVENT_DEV_CAP_BURST_MODE |
606 RTE_EVENT_DEV_CAP_EVENT_QOS |
607 RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE|
608 RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK |
609 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT |
610 RTE_EVENT_DEV_CAP_NONSEQ_MODE |
611 RTE_EVENT_DEV_CAP_CARRY_FLOW_ID |
612 RTE_EVENT_DEV_CAP_MAINTENANCE_FREE),
615 *info = evdev_sw_info;
619 sw_dump(struct rte_eventdev *dev, FILE *f)
621 const struct sw_evdev *sw = sw_pmd_priv(dev);
623 static const char * const q_type_strings[] = {
624 "Ordered", "Atomic", "Parallel", "Directed"
627 fprintf(f, "EventDev %s: ports %d, qids %d\n", "todo-fix-name",
628 sw->port_count, sw->qid_count);
630 fprintf(f, "\trx %"PRIu64"\n\tdrop %"PRIu64"\n\ttx %"PRIu64"\n",
631 sw->stats.rx_pkts, sw->stats.rx_dropped, sw->stats.tx_pkts);
632 fprintf(f, "\tsched calls: %"PRIu64"\n", sw->sched_called);
633 fprintf(f, "\tsched cq/qid call: %"PRIu64"\n", sw->sched_cq_qid_called);
634 fprintf(f, "\tsched no IQ enq: %"PRIu64"\n", sw->sched_no_iq_enqueues);
635 fprintf(f, "\tsched no CQ enq: %"PRIu64"\n", sw->sched_no_cq_enqueues);
636 uint32_t inflights = rte_atomic32_read(&sw->inflights);
637 uint32_t credits = sw->nb_events_limit - inflights;
638 fprintf(f, "\tinflight %d, credits: %d\n", inflights, credits);
640 #define COL_RED "\x1b[31m"
641 #define COL_RESET "\x1b[0m"
643 for (i = 0; i < sw->port_count; i++) {
645 const struct sw_port *p = &sw->ports[i];
646 if (!p->initialized) {
647 fprintf(f, " %sPort %d not initialized.%s\n",
648 COL_RED, i, COL_RESET);
651 fprintf(f, " Port %d %s\n", i,
652 p->is_directed ? " (SingleCons)" : "");
653 fprintf(f, "\trx %"PRIu64"\tdrop %"PRIu64"\ttx %"PRIu64
654 "\t%sinflight %d%s\n", sw->ports[i].stats.rx_pkts,
655 sw->ports[i].stats.rx_dropped,
656 sw->ports[i].stats.tx_pkts,
657 (p->inflights == p->inflight_max) ?
659 sw->ports[i].inflights, COL_RESET);
661 fprintf(f, "\tMax New: %u"
662 "\tAvg cycles PP: %"PRIu64"\tCredits: %u\n",
663 sw->ports[i].inflight_max,
664 sw->ports[i].avg_pkt_ticks,
665 sw->ports[i].inflight_credits);
666 fprintf(f, "\tReceive burst distribution:\n");
667 float zp_percent = p->zero_polls * 100.0 / p->total_polls;
668 fprintf(f, zp_percent < 10 ? "\t\t0:%.02f%% " : "\t\t0:%.0f%% ",
670 for (max = (int)RTE_DIM(p->poll_buckets); max-- > 0;)
671 if (p->poll_buckets[max] != 0)
673 for (j = 0; j <= max; j++) {
674 if (p->poll_buckets[j] != 0) {
675 float poll_pc = p->poll_buckets[j] * 100.0 /
677 fprintf(f, "%u-%u:%.02f%% ",
678 ((j << SW_DEQ_STAT_BUCKET_SHIFT) + 1),
679 ((j+1) << SW_DEQ_STAT_BUCKET_SHIFT),
685 if (p->rx_worker_ring) {
686 uint64_t used = rte_event_ring_count(p->rx_worker_ring);
687 uint64_t space = rte_event_ring_free_count(
689 const char *col = (space == 0) ? COL_RED : COL_RESET;
690 fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4"
691 PRIu64 COL_RESET"\n", col, used, space);
693 fprintf(f, "\trx ring not initialized.\n");
695 if (p->cq_worker_ring) {
696 uint64_t used = rte_event_ring_count(p->cq_worker_ring);
697 uint64_t space = rte_event_ring_free_count(
699 const char *col = (space == 0) ? COL_RED : COL_RESET;
700 fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4"
701 PRIu64 COL_RESET"\n", col, used, space);
703 fprintf(f, "\tcq ring not initialized.\n");
706 for (i = 0; i < sw->qid_count; i++) {
707 const struct sw_qid *qid = &sw->qids[i];
708 if (!qid->initialized) {
709 fprintf(f, " %sQueue %d not initialized.%s\n",
710 COL_RED, i, COL_RESET);
713 int affinities_per_port[SW_PORTS_MAX] = {0};
715 fprintf(f, " Queue %d (%s)\n", i, q_type_strings[qid->type]);
716 fprintf(f, "\trx %"PRIu64"\tdrop %"PRIu64"\ttx %"PRIu64"\n",
717 qid->stats.rx_pkts, qid->stats.rx_dropped,
719 if (qid->type == RTE_SCHED_TYPE_ORDERED) {
720 struct rob_ring *rob_buf_free =
721 qid->reorder_buffer_freelist;
723 fprintf(f, "\tReorder entries in use: %u\n",
724 rob_ring_free_count(rob_buf_free));
727 "\tReorder buffer not initialized\n");
731 for (flow = 0; flow < RTE_DIM(qid->fids); flow++)
732 if (qid->fids[flow].cq != -1) {
733 affinities_per_port[qid->fids[flow].cq]++;
737 fprintf(f, "\tPer Port Stats:\n");
738 for (port = 0; port < sw->port_count; port++) {
739 fprintf(f, "\t Port %d: Pkts: %"PRIu64, port,
741 fprintf(f, "\tFlows: %d\n", affinities_per_port[port]);
745 uint32_t iq_printed = 0;
746 for (iq = 0; iq < SW_IQS_MAX; iq++) {
747 if (!qid->iq[iq].head) {
748 fprintf(f, "\tiq %d is not initialized.\n", iq);
752 uint32_t used = iq_count(&qid->iq[iq]);
753 const char *col = COL_RESET;
755 fprintf(f, "\t%siq %d: Used %d"
756 COL_RESET"\n", col, iq, used);
761 fprintf(f, "\t-- iqs empty --\n");
766 sw_start(struct rte_eventdev *dev)
769 struct sw_evdev *sw = sw_pmd_priv(dev);
771 rte_service_component_runstate_set(sw->service_id, 1);
773 /* check a service core is mapped to this service */
774 if (!rte_service_runstate_get(sw->service_id)) {
775 SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
780 /* check all ports are set up */
781 for (i = 0; i < sw->port_count; i++)
782 if (sw->ports[i].rx_worker_ring == NULL) {
783 SW_LOG_ERR("Port %d not configured\n", i);
787 /* check all queues are configured and mapped to ports*/
788 for (i = 0; i < sw->qid_count; i++)
789 if (!sw->qids[i].initialized ||
790 sw->qids[i].cq_num_mapped_cqs == 0) {
791 SW_LOG_ERR("Queue %d not configured\n", i);
795 /* build up our prioritized array of qids */
796 /* We don't use qsort here, as if all/multiple entries have the same
797 * priority, the result is non-deterministic. From "man 3 qsort":
798 * "If two members compare as equal, their order in the sorted
799 * array is undefined."
802 for (j = 0; j <= RTE_EVENT_DEV_PRIORITY_LOWEST; j++) {
803 for (i = 0; i < sw->qid_count; i++) {
804 if (sw->qids[i].priority == j) {
805 sw->qids_prioritized[qidx] = &sw->qids[i];
813 if (sw_xstats_init(sw) < 0)
823 sw_stop(struct rte_eventdev *dev)
825 struct sw_evdev *sw = sw_pmd_priv(dev);
828 /* Stop the scheduler if it's running */
829 runstate = rte_service_runstate_get(sw->service_id);
831 rte_service_runstate_set(sw->service_id, 0);
833 while (rte_service_may_be_active(sw->service_id))
836 /* Flush all events out of the device */
837 while (!(sw_qids_empty(sw) && sw_ports_empty(sw))) {
838 sw_event_schedule(dev);
840 sw_drain_queues(dev);
843 sw_clean_qid_iqs(dev);
844 sw_xstats_uninit(sw);
849 rte_service_runstate_set(sw->service_id, 1);
853 sw_close(struct rte_eventdev *dev)
855 struct sw_evdev *sw = sw_pmd_priv(dev);
858 for (i = 0; i < sw->qid_count; i++)
859 sw_queue_release(dev, i);
862 for (i = 0; i < sw->port_count; i++)
863 sw_port_release(&sw->ports[i]);
866 memset(&sw->stats, 0, sizeof(sw->stats));
867 sw->sched_called = 0;
868 sw->sched_no_iq_enqueues = 0;
869 sw->sched_no_cq_enqueues = 0;
870 sw->sched_cq_qid_called = 0;
876 assign_numa_node(const char *key __rte_unused, const char *value, void *opaque)
878 int *socket_id = opaque;
879 *socket_id = atoi(value);
880 if (*socket_id >= RTE_MAX_NUMA_NODES)
886 set_sched_quanta(const char *key __rte_unused, const char *value, void *opaque)
888 int *quanta = opaque;
889 *quanta = atoi(value);
890 if (*quanta < 0 || *quanta >= 4096)
896 set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
898 int *credit = opaque;
899 *credit = atoi(value);
900 if (*credit < 0 || *credit >= 128)
906 set_deq_burst_sz(const char *key __rte_unused, const char *value, void *opaque)
908 int *deq_burst_sz = opaque;
909 *deq_burst_sz = atoi(value);
910 if (*deq_burst_sz < 0 || *deq_burst_sz > SCHED_DEQUEUE_MAX_BURST_SIZE)
916 set_min_burst_sz(const char *key __rte_unused, const char *value, void *opaque)
918 int *min_burst_sz = opaque;
919 *min_burst_sz = atoi(value);
920 if (*min_burst_sz < 0 || *min_burst_sz > SCHED_DEQUEUE_MAX_BURST_SIZE)
926 set_refill_once(const char *key __rte_unused, const char *value, void *opaque)
928 int *refill_once_per_call = opaque;
929 *refill_once_per_call = atoi(value);
930 if (*refill_once_per_call < 0 || *refill_once_per_call > 1)
935 static int32_t sw_sched_service_func(void *args)
937 struct rte_eventdev *dev = args;
938 sw_event_schedule(dev);
943 sw_probe(struct rte_vdev_device *vdev)
945 static struct eventdev_ops evdev_sw_ops = {
946 .dev_configure = sw_dev_configure,
947 .dev_infos_get = sw_info_get,
948 .dev_close = sw_close,
949 .dev_start = sw_start,
953 .queue_def_conf = sw_queue_def_conf,
954 .queue_setup = sw_queue_setup,
955 .queue_release = sw_queue_release,
956 .port_def_conf = sw_port_def_conf,
957 .port_setup = sw_port_setup,
958 .port_release = sw_port_release,
959 .port_link = sw_port_link,
960 .port_unlink = sw_port_unlink,
961 .port_unlinks_in_progress = sw_port_unlinks_in_progress,
963 .eth_rx_adapter_caps_get = sw_eth_rx_adapter_caps_get,
965 .timer_adapter_caps_get = sw_timer_adapter_caps_get,
967 .crypto_adapter_caps_get = sw_crypto_adapter_caps_get,
969 .xstats_get = sw_xstats_get,
970 .xstats_get_names = sw_xstats_get_names,
971 .xstats_get_by_name = sw_xstats_get_by_name,
972 .xstats_reset = sw_xstats_reset,
974 .dev_selftest = test_sw_eventdev,
977 static const char *const args[] = {
988 struct rte_eventdev *dev;
990 int socket_id = rte_socket_id();
991 int sched_quanta = SW_DEFAULT_SCHED_QUANTA;
992 int credit_quanta = SW_DEFAULT_CREDIT_QUANTA;
993 int min_burst_size = 1;
994 int deq_burst_size = SCHED_DEQUEUE_DEFAULT_BURST_SIZE;
997 name = rte_vdev_device_name(vdev);
998 params = rte_vdev_device_args(vdev);
999 if (params != NULL && params[0] != '\0') {
1000 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
1004 "Ignoring unsupported parameters when creating device '%s'\n",
1007 int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
1008 assign_numa_node, &socket_id);
1011 "%s: Error parsing numa node parameter",
1013 rte_kvargs_free(kvlist);
1017 ret = rte_kvargs_process(kvlist, SCHED_QUANTA_ARG,
1018 set_sched_quanta, &sched_quanta);
1021 "%s: Error parsing sched quanta parameter",
1023 rte_kvargs_free(kvlist);
1027 ret = rte_kvargs_process(kvlist, CREDIT_QUANTA_ARG,
1028 set_credit_quanta, &credit_quanta);
1031 "%s: Error parsing credit quanta parameter",
1033 rte_kvargs_free(kvlist);
1037 ret = rte_kvargs_process(kvlist, MIN_BURST_SIZE_ARG,
1038 set_min_burst_sz, &min_burst_size);
1041 "%s: Error parsing minimum burst size parameter",
1043 rte_kvargs_free(kvlist);
1047 ret = rte_kvargs_process(kvlist, DEQ_BURST_SIZE_ARG,
1048 set_deq_burst_sz, &deq_burst_size);
1051 "%s: Error parsing dequeue burst size parameter",
1053 rte_kvargs_free(kvlist);
1057 ret = rte_kvargs_process(kvlist, REFIL_ONCE_ARG,
1058 set_refill_once, &refill_once);
1061 "%s: Error parsing refill once per call switch",
1063 rte_kvargs_free(kvlist);
1067 rte_kvargs_free(kvlist);
1072 "Creating eventdev sw device %s, numa_node=%d, "
1073 "sched_quanta=%d, credit_quanta=%d "
1074 "min_burst=%d, deq_burst=%d, refill_once=%d\n",
1075 name, socket_id, sched_quanta, credit_quanta,
1076 min_burst_size, deq_burst_size, refill_once);
1078 dev = rte_event_pmd_vdev_init(name,
1079 sizeof(struct sw_evdev), socket_id);
1081 SW_LOG_ERR("eventdev vdev init() failed");
1084 dev->dev_ops = &evdev_sw_ops;
1085 dev->enqueue = sw_event_enqueue;
1086 dev->enqueue_burst = sw_event_enqueue_burst;
1087 dev->enqueue_new_burst = sw_event_enqueue_burst;
1088 dev->enqueue_forward_burst = sw_event_enqueue_burst;
1089 dev->dequeue = sw_event_dequeue;
1090 dev->dequeue_burst = sw_event_dequeue_burst;
1092 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1095 sw = dev->data->dev_private;
1096 sw->data = dev->data;
1098 /* copy values passed from vdev command line to instance */
1099 sw->credit_update_quanta = credit_quanta;
1100 sw->sched_quanta = sched_quanta;
1101 sw->sched_min_burst_size = min_burst_size;
1102 sw->sched_deq_burst_size = deq_burst_size;
1103 sw->refill_once_per_iter = refill_once;
1105 /* register service with EAL */
1106 struct rte_service_spec service;
1107 memset(&service, 0, sizeof(struct rte_service_spec));
1108 snprintf(service.name, sizeof(service.name), "%s_service", name);
1109 snprintf(sw->service_name, sizeof(sw->service_name), "%s_service",
1111 service.socket_id = socket_id;
1112 service.callback = sw_sched_service_func;
1113 service.callback_userdata = (void *)dev;
1115 int32_t ret = rte_service_component_register(&service, &sw->service_id);
1117 SW_LOG_ERR("service register() failed");
1121 dev->data->service_inited = 1;
1122 dev->data->service_id = sw->service_id;
1124 event_dev_probing_finish(dev);
1130 sw_remove(struct rte_vdev_device *vdev)
1134 name = rte_vdev_device_name(vdev);
1138 SW_LOG_INFO("Closing eventdev sw device %s\n", name);
1140 return rte_event_pmd_vdev_uninit(name);
1143 static struct rte_vdev_driver evdev_sw_pmd_drv = {
1148 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SW_PMD, evdev_sw_pmd_drv);
1149 RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "=<int> "
1150 SCHED_QUANTA_ARG "=<int>" CREDIT_QUANTA_ARG "=<int>"
1151 MIN_BURST_SIZE_ARG "=<int>" DEQ_BURST_SIZE_ARG "=<int>"
1152 REFIL_ONCE_ARG "=<int>");
1153 RTE_LOG_REGISTER_DEFAULT(eventdev_sw_log_level, NOTICE);