net/mlx4: fix RxQ errors stat
[dpdk.git] / drivers / event / sw / sw_evdev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2016-2017 Intel Corporation
3  */
4
5 #include <inttypes.h>
6 #include <string.h>
7
8 #include <rte_bus_vdev.h>
9 #include <rte_kvargs.h>
10 #include <rte_ring.h>
11 #include <rte_errno.h>
12 #include <rte_event_ring.h>
13 #include <rte_service_component.h>
14
15 #include "sw_evdev.h"
16 #include "iq_chunk.h"
17
18 #define EVENTDEV_NAME_SW_PMD event_sw
19 #define NUMA_NODE_ARG "numa_node"
20 #define SCHED_QUANTA_ARG "sched_quanta"
21 #define CREDIT_QUANTA_ARG "credit_quanta"
22
23 static void
24 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info);
25
26 static int
27 sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[],
28                 const uint8_t priorities[], uint16_t num)
29 {
30         struct sw_port *p = port;
31         struct sw_evdev *sw = sw_pmd_priv(dev);
32         int i;
33
34         RTE_SET_USED(priorities);
35         for (i = 0; i < num; i++) {
36                 struct sw_qid *q = &sw->qids[queues[i]];
37                 unsigned int j;
38
39                 /* check for qid map overflow */
40                 if (q->cq_num_mapped_cqs >= RTE_DIM(q->cq_map)) {
41                         rte_errno = -EDQUOT;
42                         break;
43                 }
44
45                 if (p->is_directed && p->num_qids_mapped > 0) {
46                         rte_errno = -EDQUOT;
47                         break;
48                 }
49
50                 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
51                         if (q->cq_map[j] == p->id)
52                                 break;
53                 }
54
55                 /* check if port is already linked */
56                 if (j < q->cq_num_mapped_cqs)
57                         continue;
58
59                 if (q->type == SW_SCHED_TYPE_DIRECT) {
60                         /* check directed qids only map to one port */
61                         if (p->num_qids_mapped > 0) {
62                                 rte_errno = -EDQUOT;
63                                 break;
64                         }
65                         /* check port only takes a directed flow */
66                         if (num > 1) {
67                                 rte_errno = -EDQUOT;
68                                 break;
69                         }
70
71                         p->is_directed = 1;
72                         p->num_qids_mapped = 1;
73                 } else if (q->type == RTE_SCHED_TYPE_ORDERED) {
74                         p->num_ordered_qids++;
75                         p->num_qids_mapped++;
76                 } else if (q->type == RTE_SCHED_TYPE_ATOMIC ||
77                                 q->type == RTE_SCHED_TYPE_PARALLEL) {
78                         p->num_qids_mapped++;
79                 }
80
81                 q->cq_map[q->cq_num_mapped_cqs] = p->id;
82                 rte_smp_wmb();
83                 q->cq_num_mapped_cqs++;
84         }
85         return i;
86 }
87
88 static int
89 sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[],
90                 uint16_t nb_unlinks)
91 {
92         struct sw_port *p = port;
93         struct sw_evdev *sw = sw_pmd_priv(dev);
94         unsigned int i, j;
95
96         int unlinked = 0;
97         for (i = 0; i < nb_unlinks; i++) {
98                 struct sw_qid *q = &sw->qids[queues[i]];
99                 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
100                         if (q->cq_map[j] == p->id) {
101                                 q->cq_map[j] =
102                                         q->cq_map[q->cq_num_mapped_cqs - 1];
103                                 rte_smp_wmb();
104                                 q->cq_num_mapped_cqs--;
105                                 unlinked++;
106
107                                 p->num_qids_mapped--;
108
109                                 if (q->type == RTE_SCHED_TYPE_ORDERED)
110                                         p->num_ordered_qids--;
111
112                                 continue;
113                         }
114                 }
115         }
116
117         p->unlinks_in_progress += unlinked;
118         rte_smp_mb();
119
120         return unlinked;
121 }
122
123 static int
124 sw_port_unlinks_in_progress(struct rte_eventdev *dev, void *port)
125 {
126         RTE_SET_USED(dev);
127         struct sw_port *p = port;
128         return p->unlinks_in_progress;
129 }
130
131 static int
132 sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
133                 const struct rte_event_port_conf *conf)
134 {
135         struct sw_evdev *sw = sw_pmd_priv(dev);
136         struct sw_port *p = &sw->ports[port_id];
137         char buf[RTE_RING_NAMESIZE];
138         unsigned int i;
139
140         struct rte_event_dev_info info;
141         sw_info_get(dev, &info);
142
143         /* detect re-configuring and return credits to instance if needed */
144         if (p->initialized) {
145                 /* taking credits from pool is done one quanta at a time, and
146                  * credits may be spend (counted in p->inflights) or still
147                  * available in the port (p->inflight_credits). We must return
148                  * the sum to no leak credits
149                  */
150                 int possible_inflights = p->inflight_credits + p->inflights;
151                 rte_atomic32_sub(&sw->inflights, possible_inflights);
152         }
153
154         *p = (struct sw_port){0}; /* zero entire structure */
155         p->id = port_id;
156         p->sw = sw;
157
158         /* check to see if rings exists - port_setup() can be called multiple
159          * times legally (assuming device is stopped). If ring exists, free it
160          * to so it gets re-created with the correct size
161          */
162         snprintf(buf, sizeof(buf), "sw%d_p%u_%s", dev->data->dev_id,
163                         port_id, "rx_worker_ring");
164         struct rte_event_ring *existing_ring = rte_event_ring_lookup(buf);
165         if (existing_ring)
166                 rte_event_ring_free(existing_ring);
167
168         p->rx_worker_ring = rte_event_ring_create(buf, MAX_SW_PROD_Q_DEPTH,
169                         dev->data->socket_id,
170                         RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
171         if (p->rx_worker_ring == NULL) {
172                 SW_LOG_ERR("Error creating RX worker ring for port %d\n",
173                                 port_id);
174                 return -1;
175         }
176
177         p->inflight_max = conf->new_event_threshold;
178         p->implicit_release = !conf->disable_implicit_release;
179
180         /* check if ring exists, same as rx_worker above */
181         snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id,
182                         port_id, "cq_worker_ring");
183         existing_ring = rte_event_ring_lookup(buf);
184         if (existing_ring)
185                 rte_event_ring_free(existing_ring);
186
187         p->cq_worker_ring = rte_event_ring_create(buf, conf->dequeue_depth,
188                         dev->data->socket_id,
189                         RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
190         if (p->cq_worker_ring == NULL) {
191                 rte_event_ring_free(p->rx_worker_ring);
192                 SW_LOG_ERR("Error creating CQ worker ring for port %d\n",
193                                 port_id);
194                 return -1;
195         }
196         sw->cq_ring_space[port_id] = conf->dequeue_depth;
197
198         /* set hist list contents to empty */
199         for (i = 0; i < SW_PORT_HIST_LIST; i++) {
200                 p->hist_list[i].fid = -1;
201                 p->hist_list[i].qid = -1;
202         }
203         dev->data->ports[port_id] = p;
204
205         rte_smp_wmb();
206         p->initialized = 1;
207         return 0;
208 }
209
210 static void
211 sw_port_release(void *port)
212 {
213         struct sw_port *p = (void *)port;
214         if (p == NULL)
215                 return;
216
217         rte_event_ring_free(p->rx_worker_ring);
218         rte_event_ring_free(p->cq_worker_ring);
219         memset(p, 0, sizeof(*p));
220 }
221
222 static int32_t
223 qid_init(struct sw_evdev *sw, unsigned int idx, int type,
224                 const struct rte_event_queue_conf *queue_conf)
225 {
226         unsigned int i;
227         int dev_id = sw->data->dev_id;
228         int socket_id = sw->data->socket_id;
229         char buf[IQ_ROB_NAMESIZE];
230         struct sw_qid *qid = &sw->qids[idx];
231
232         /* Initialize the FID structures to no pinning (-1), and zero packets */
233         const struct sw_fid_t fid = {.cq = -1, .pcount = 0};
234         for (i = 0; i < RTE_DIM(qid->fids); i++)
235                 qid->fids[i] = fid;
236
237         qid->id = idx;
238         qid->type = type;
239         qid->priority = queue_conf->priority;
240
241         if (qid->type == RTE_SCHED_TYPE_ORDERED) {
242                 char ring_name[RTE_RING_NAMESIZE];
243                 uint32_t window_size;
244
245                 /* rte_ring and window_size_mask require require window_size to
246                  * be a power-of-2.
247                  */
248                 window_size = rte_align32pow2(
249                                 queue_conf->nb_atomic_order_sequences);
250
251                 qid->window_size = window_size - 1;
252
253                 if (!window_size) {
254                         SW_LOG_DBG(
255                                 "invalid reorder_window_size for ordered queue\n"
256                                 );
257                         goto cleanup;
258                 }
259
260                 snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i);
261                 qid->reorder_buffer = rte_zmalloc_socket(buf,
262                                 window_size * sizeof(qid->reorder_buffer[0]),
263                                 0, socket_id);
264                 if (!qid->reorder_buffer) {
265                         SW_LOG_DBG("reorder_buffer malloc failed\n");
266                         goto cleanup;
267                 }
268
269                 memset(&qid->reorder_buffer[0],
270                        0,
271                        window_size * sizeof(qid->reorder_buffer[0]));
272
273                 snprintf(ring_name, sizeof(ring_name), "sw%d_q%d_freelist",
274                                 dev_id, idx);
275
276                 /* lookup the ring, and if it already exists, free it */
277                 struct rte_ring *cleanup = rte_ring_lookup(ring_name);
278                 if (cleanup)
279                         rte_ring_free(cleanup);
280
281                 qid->reorder_buffer_freelist = rte_ring_create(ring_name,
282                                 window_size,
283                                 socket_id,
284                                 RING_F_SP_ENQ | RING_F_SC_DEQ);
285                 if (!qid->reorder_buffer_freelist) {
286                         SW_LOG_DBG("freelist ring create failed");
287                         goto cleanup;
288                 }
289
290                 /* Populate the freelist with reorder buffer entries. Enqueue
291                  * 'window_size - 1' entries because the rte_ring holds only
292                  * that many.
293                  */
294                 for (i = 0; i < window_size - 1; i++) {
295                         if (rte_ring_sp_enqueue(qid->reorder_buffer_freelist,
296                                                 &qid->reorder_buffer[i]) < 0)
297                                 goto cleanup;
298                 }
299
300                 qid->reorder_buffer_index = 0;
301                 qid->cq_next_tx = 0;
302         }
303
304         qid->initialized = 1;
305
306         return 0;
307
308 cleanup:
309         if (qid->reorder_buffer) {
310                 rte_free(qid->reorder_buffer);
311                 qid->reorder_buffer = NULL;
312         }
313
314         if (qid->reorder_buffer_freelist) {
315                 rte_ring_free(qid->reorder_buffer_freelist);
316                 qid->reorder_buffer_freelist = NULL;
317         }
318
319         return -EINVAL;
320 }
321
322 static void
323 sw_queue_release(struct rte_eventdev *dev, uint8_t id)
324 {
325         struct sw_evdev *sw = sw_pmd_priv(dev);
326         struct sw_qid *qid = &sw->qids[id];
327
328         if (qid->type == RTE_SCHED_TYPE_ORDERED) {
329                 rte_free(qid->reorder_buffer);
330                 rte_ring_free(qid->reorder_buffer_freelist);
331         }
332         memset(qid, 0, sizeof(*qid));
333 }
334
335 static int
336 sw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
337                 const struct rte_event_queue_conf *conf)
338 {
339         int type;
340
341         type = conf->schedule_type;
342
343         if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) {
344                 type = SW_SCHED_TYPE_DIRECT;
345         } else if (RTE_EVENT_QUEUE_CFG_ALL_TYPES
346                         & conf->event_queue_cfg) {
347                 SW_LOG_ERR("QUEUE_CFG_ALL_TYPES not supported\n");
348                 return -ENOTSUP;
349         }
350
351         struct sw_evdev *sw = sw_pmd_priv(dev);
352
353         if (sw->qids[queue_id].initialized)
354                 sw_queue_release(dev, queue_id);
355
356         return qid_init(sw, queue_id, type, conf);
357 }
358
359 static void
360 sw_init_qid_iqs(struct sw_evdev *sw)
361 {
362         int i, j;
363
364         /* Initialize the IQ memory of all configured qids */
365         for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) {
366                 struct sw_qid *qid = &sw->qids[i];
367
368                 if (!qid->initialized)
369                         continue;
370
371                 for (j = 0; j < SW_IQS_MAX; j++)
372                         iq_init(sw, &qid->iq[j]);
373         }
374 }
375
376 static int
377 sw_qids_empty(struct sw_evdev *sw)
378 {
379         unsigned int i, j;
380
381         for (i = 0; i < sw->qid_count; i++) {
382                 for (j = 0; j < SW_IQS_MAX; j++) {
383                         if (iq_count(&sw->qids[i].iq[j]))
384                                 return 0;
385                 }
386         }
387
388         return 1;
389 }
390
391 static int
392 sw_ports_empty(struct sw_evdev *sw)
393 {
394         unsigned int i;
395
396         for (i = 0; i < sw->port_count; i++) {
397                 if ((rte_event_ring_count(sw->ports[i].rx_worker_ring)) ||
398                      rte_event_ring_count(sw->ports[i].cq_worker_ring))
399                         return 0;
400         }
401
402         return 1;
403 }
404
405 static void
406 sw_drain_ports(struct rte_eventdev *dev)
407 {
408         struct sw_evdev *sw = sw_pmd_priv(dev);
409         eventdev_stop_flush_t flush;
410         unsigned int i;
411         uint8_t dev_id;
412         void *arg;
413
414         flush = dev->dev_ops->dev_stop_flush;
415         dev_id = dev->data->dev_id;
416         arg = dev->data->dev_stop_flush_arg;
417
418         for (i = 0; i < sw->port_count; i++) {
419                 struct rte_event ev;
420
421                 while (rte_event_dequeue_burst(dev_id, i, &ev, 1, 0)) {
422                         if (flush)
423                                 flush(dev_id, ev, arg);
424
425                         ev.op = RTE_EVENT_OP_RELEASE;
426                         rte_event_enqueue_burst(dev_id, i, &ev, 1);
427                 }
428         }
429 }
430
431 static void
432 sw_drain_queue(struct rte_eventdev *dev, struct sw_iq *iq)
433 {
434         struct sw_evdev *sw = sw_pmd_priv(dev);
435         eventdev_stop_flush_t flush;
436         uint8_t dev_id;
437         void *arg;
438
439         flush = dev->dev_ops->dev_stop_flush;
440         dev_id = dev->data->dev_id;
441         arg = dev->data->dev_stop_flush_arg;
442
443         while (iq_count(iq) > 0) {
444                 struct rte_event ev;
445
446                 iq_dequeue_burst(sw, iq, &ev, 1);
447
448                 if (flush)
449                         flush(dev_id, ev, arg);
450         }
451 }
452
453 static void
454 sw_drain_queues(struct rte_eventdev *dev)
455 {
456         struct sw_evdev *sw = sw_pmd_priv(dev);
457         unsigned int i, j;
458
459         for (i = 0; i < sw->qid_count; i++) {
460                 for (j = 0; j < SW_IQS_MAX; j++)
461                         sw_drain_queue(dev, &sw->qids[i].iq[j]);
462         }
463 }
464
465 static void
466 sw_clean_qid_iqs(struct rte_eventdev *dev)
467 {
468         struct sw_evdev *sw = sw_pmd_priv(dev);
469         int i, j;
470
471         /* Release the IQ memory of all configured qids */
472         for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) {
473                 struct sw_qid *qid = &sw->qids[i];
474
475                 for (j = 0; j < SW_IQS_MAX; j++) {
476                         if (!qid->iq[j].head)
477                                 continue;
478                         iq_free_chunk_list(sw, qid->iq[j].head);
479                         qid->iq[j].head = NULL;
480                 }
481         }
482 }
483
484 static void
485 sw_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id,
486                                  struct rte_event_queue_conf *conf)
487 {
488         RTE_SET_USED(dev);
489         RTE_SET_USED(queue_id);
490
491         static const struct rte_event_queue_conf default_conf = {
492                 .nb_atomic_flows = 4096,
493                 .nb_atomic_order_sequences = 1,
494                 .schedule_type = RTE_SCHED_TYPE_ATOMIC,
495                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
496         };
497
498         *conf = default_conf;
499 }
500
501 static void
502 sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
503                  struct rte_event_port_conf *port_conf)
504 {
505         RTE_SET_USED(dev);
506         RTE_SET_USED(port_id);
507
508         port_conf->new_event_threshold = 1024;
509         port_conf->dequeue_depth = 16;
510         port_conf->enqueue_depth = 16;
511         port_conf->disable_implicit_release = 0;
512 }
513
514 static int
515 sw_dev_configure(const struct rte_eventdev *dev)
516 {
517         struct sw_evdev *sw = sw_pmd_priv(dev);
518         const struct rte_eventdev_data *data = dev->data;
519         const struct rte_event_dev_config *conf = &data->dev_conf;
520         int num_chunks, i;
521
522         sw->qid_count = conf->nb_event_queues;
523         sw->port_count = conf->nb_event_ports;
524         sw->nb_events_limit = conf->nb_events_limit;
525         rte_atomic32_set(&sw->inflights, 0);
526
527         /* Number of chunks sized for worst-case spread of events across IQs */
528         num_chunks = ((SW_INFLIGHT_EVENTS_TOTAL/SW_EVS_PER_Q_CHUNK)+1) +
529                         sw->qid_count*SW_IQS_MAX*2;
530
531         /* If this is a reconfiguration, free the previous IQ allocation. All
532          * IQ chunk references were cleaned out of the QIDs in sw_stop(), and
533          * will be reinitialized in sw_start().
534          */
535         if (sw->chunks)
536                 rte_free(sw->chunks);
537
538         sw->chunks = rte_malloc_socket(NULL,
539                                        sizeof(struct sw_queue_chunk) *
540                                        num_chunks,
541                                        0,
542                                        sw->data->socket_id);
543         if (!sw->chunks)
544                 return -ENOMEM;
545
546         sw->chunk_list_head = NULL;
547         for (i = 0; i < num_chunks; i++)
548                 iq_free_chunk(sw, &sw->chunks[i]);
549
550         if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
551                 return -ENOTSUP;
552
553         return 0;
554 }
555
556 struct rte_eth_dev;
557
558 static int
559 sw_eth_rx_adapter_caps_get(const struct rte_eventdev *dev,
560                         const struct rte_eth_dev *eth_dev,
561                         uint32_t *caps)
562 {
563         RTE_SET_USED(dev);
564         RTE_SET_USED(eth_dev);
565         *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP;
566         return 0;
567 }
568
569 static int
570 sw_timer_adapter_caps_get(const struct rte_eventdev *dev,
571                           uint64_t flags,
572                           uint32_t *caps,
573                           const struct rte_event_timer_adapter_ops **ops)
574 {
575         RTE_SET_USED(dev);
576         RTE_SET_USED(flags);
577         *caps = 0;
578
579         /* Use default SW ops */
580         *ops = NULL;
581
582         return 0;
583 }
584
585 static int
586 sw_crypto_adapter_caps_get(const struct rte_eventdev *dev,
587                            const struct rte_cryptodev *cdev,
588                            uint32_t *caps)
589 {
590         RTE_SET_USED(dev);
591         RTE_SET_USED(cdev);
592         *caps = RTE_EVENT_CRYPTO_ADAPTER_SW_CAP;
593         return 0;
594 }
595
596 static void
597 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info)
598 {
599         RTE_SET_USED(dev);
600
601         static const struct rte_event_dev_info evdev_sw_info = {
602                         .driver_name = SW_PMD_NAME,
603                         .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
604                         .max_event_queue_flows = SW_QID_NUM_FIDS,
605                         .max_event_queue_priority_levels = SW_Q_PRIORITY_MAX,
606                         .max_event_priority_levels = SW_IQS_MAX,
607                         .max_event_ports = SW_PORTS_MAX,
608                         .max_event_port_dequeue_depth = MAX_SW_CONS_Q_DEPTH,
609                         .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH,
610                         .max_num_events = SW_INFLIGHT_EVENTS_TOTAL,
611                         .event_dev_cap = (
612                                 RTE_EVENT_DEV_CAP_QUEUE_QOS |
613                                 RTE_EVENT_DEV_CAP_BURST_MODE |
614                                 RTE_EVENT_DEV_CAP_EVENT_QOS |
615                                 RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE|
616                                 RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK |
617                                 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT |
618                                 RTE_EVENT_DEV_CAP_NONSEQ_MODE),
619         };
620
621         *info = evdev_sw_info;
622 }
623
624 static void
625 sw_dump(struct rte_eventdev *dev, FILE *f)
626 {
627         const struct sw_evdev *sw = sw_pmd_priv(dev);
628
629         static const char * const q_type_strings[] = {
630                         "Ordered", "Atomic", "Parallel", "Directed"
631         };
632         uint32_t i;
633         fprintf(f, "EventDev %s: ports %d, qids %d\n", "todo-fix-name",
634                         sw->port_count, sw->qid_count);
635
636         fprintf(f, "\trx   %"PRIu64"\n\tdrop %"PRIu64"\n\ttx   %"PRIu64"\n",
637                 sw->stats.rx_pkts, sw->stats.rx_dropped, sw->stats.tx_pkts);
638         fprintf(f, "\tsched calls: %"PRIu64"\n", sw->sched_called);
639         fprintf(f, "\tsched cq/qid call: %"PRIu64"\n", sw->sched_cq_qid_called);
640         fprintf(f, "\tsched no IQ enq: %"PRIu64"\n", sw->sched_no_iq_enqueues);
641         fprintf(f, "\tsched no CQ enq: %"PRIu64"\n", sw->sched_no_cq_enqueues);
642         uint32_t inflights = rte_atomic32_read(&sw->inflights);
643         uint32_t credits = sw->nb_events_limit - inflights;
644         fprintf(f, "\tinflight %d, credits: %d\n", inflights, credits);
645
646 #define COL_RED "\x1b[31m"
647 #define COL_RESET "\x1b[0m"
648
649         for (i = 0; i < sw->port_count; i++) {
650                 int max, j;
651                 const struct sw_port *p = &sw->ports[i];
652                 if (!p->initialized) {
653                         fprintf(f, "  %sPort %d not initialized.%s\n",
654                                 COL_RED, i, COL_RESET);
655                         continue;
656                 }
657                 fprintf(f, "  Port %d %s\n", i,
658                         p->is_directed ? " (SingleCons)" : "");
659                 fprintf(f, "\trx   %"PRIu64"\tdrop %"PRIu64"\ttx   %"PRIu64
660                         "\t%sinflight %d%s\n", sw->ports[i].stats.rx_pkts,
661                         sw->ports[i].stats.rx_dropped,
662                         sw->ports[i].stats.tx_pkts,
663                         (p->inflights == p->inflight_max) ?
664                                 COL_RED : COL_RESET,
665                         sw->ports[i].inflights, COL_RESET);
666
667                 fprintf(f, "\tMax New: %u"
668                         "\tAvg cycles PP: %"PRIu64"\tCredits: %u\n",
669                         sw->ports[i].inflight_max,
670                         sw->ports[i].avg_pkt_ticks,
671                         sw->ports[i].inflight_credits);
672                 fprintf(f, "\tReceive burst distribution:\n");
673                 float zp_percent = p->zero_polls * 100.0 / p->total_polls;
674                 fprintf(f, zp_percent < 10 ? "\t\t0:%.02f%% " : "\t\t0:%.0f%% ",
675                                 zp_percent);
676                 for (max = (int)RTE_DIM(p->poll_buckets); max-- > 0;)
677                         if (p->poll_buckets[max] != 0)
678                                 break;
679                 for (j = 0; j <= max; j++) {
680                         if (p->poll_buckets[j] != 0) {
681                                 float poll_pc = p->poll_buckets[j] * 100.0 /
682                                         p->total_polls;
683                                 fprintf(f, "%u-%u:%.02f%% ",
684                                         ((j << SW_DEQ_STAT_BUCKET_SHIFT) + 1),
685                                         ((j+1) << SW_DEQ_STAT_BUCKET_SHIFT),
686                                         poll_pc);
687                         }
688                 }
689                 fprintf(f, "\n");
690
691                 if (p->rx_worker_ring) {
692                         uint64_t used = rte_event_ring_count(p->rx_worker_ring);
693                         uint64_t space = rte_event_ring_free_count(
694                                         p->rx_worker_ring);
695                         const char *col = (space == 0) ? COL_RED : COL_RESET;
696                         fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4"
697                                         PRIu64 COL_RESET"\n", col, used, space);
698                 } else
699                         fprintf(f, "\trx ring not initialized.\n");
700
701                 if (p->cq_worker_ring) {
702                         uint64_t used = rte_event_ring_count(p->cq_worker_ring);
703                         uint64_t space = rte_event_ring_free_count(
704                                         p->cq_worker_ring);
705                         const char *col = (space == 0) ? COL_RED : COL_RESET;
706                         fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4"
707                                         PRIu64 COL_RESET"\n", col, used, space);
708                 } else
709                         fprintf(f, "\tcq ring not initialized.\n");
710         }
711
712         for (i = 0; i < sw->qid_count; i++) {
713                 const struct sw_qid *qid = &sw->qids[i];
714                 if (!qid->initialized) {
715                         fprintf(f, "  %sQueue %d not initialized.%s\n",
716                                 COL_RED, i, COL_RESET);
717                         continue;
718                 }
719                 int affinities_per_port[SW_PORTS_MAX] = {0};
720                 uint32_t inflights = 0;
721
722                 fprintf(f, "  Queue %d (%s)\n", i, q_type_strings[qid->type]);
723                 fprintf(f, "\trx   %"PRIu64"\tdrop %"PRIu64"\ttx   %"PRIu64"\n",
724                         qid->stats.rx_pkts, qid->stats.rx_dropped,
725                         qid->stats.tx_pkts);
726                 if (qid->type == RTE_SCHED_TYPE_ORDERED) {
727                         struct rte_ring *rob_buf_free =
728                                 qid->reorder_buffer_freelist;
729                         if (rob_buf_free)
730                                 fprintf(f, "\tReorder entries in use: %u\n",
731                                         rte_ring_free_count(rob_buf_free));
732                         else
733                                 fprintf(f,
734                                         "\tReorder buffer not initialized\n");
735                 }
736
737                 uint32_t flow;
738                 for (flow = 0; flow < RTE_DIM(qid->fids); flow++)
739                         if (qid->fids[flow].cq != -1) {
740                                 affinities_per_port[qid->fids[flow].cq]++;
741                                 inflights += qid->fids[flow].pcount;
742                         }
743
744                 uint32_t port;
745                 fprintf(f, "\tPer Port Stats:\n");
746                 for (port = 0; port < sw->port_count; port++) {
747                         fprintf(f, "\t  Port %d: Pkts: %"PRIu64, port,
748                                         qid->to_port[port]);
749                         fprintf(f, "\tFlows: %d\n", affinities_per_port[port]);
750                 }
751
752                 uint32_t iq;
753                 uint32_t iq_printed = 0;
754                 for (iq = 0; iq < SW_IQS_MAX; iq++) {
755                         if (!qid->iq[iq].head) {
756                                 fprintf(f, "\tiq %d is not initialized.\n", iq);
757                                 iq_printed = 1;
758                                 continue;
759                         }
760                         uint32_t used = iq_count(&qid->iq[iq]);
761                         const char *col = COL_RESET;
762                         if (used > 0) {
763                                 fprintf(f, "\t%siq %d: Used %d"
764                                         COL_RESET"\n", col, iq, used);
765                                 iq_printed = 1;
766                         }
767                 }
768                 if (iq_printed == 0)
769                         fprintf(f, "\t-- iqs empty --\n");
770         }
771 }
772
773 static int
774 sw_start(struct rte_eventdev *dev)
775 {
776         unsigned int i, j;
777         struct sw_evdev *sw = sw_pmd_priv(dev);
778
779         rte_service_component_runstate_set(sw->service_id, 1);
780
781         /* check a service core is mapped to this service */
782         if (!rte_service_runstate_get(sw->service_id)) {
783                 SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
784                                 sw->service_name);
785                 return -ENOENT;
786         }
787
788         /* check all ports are set up */
789         for (i = 0; i < sw->port_count; i++)
790                 if (sw->ports[i].rx_worker_ring == NULL) {
791                         SW_LOG_ERR("Port %d not configured\n", i);
792                         return -ESTALE;
793                 }
794
795         /* check all queues are configured and mapped to ports*/
796         for (i = 0; i < sw->qid_count; i++)
797                 if (!sw->qids[i].initialized ||
798                     sw->qids[i].cq_num_mapped_cqs == 0) {
799                         SW_LOG_ERR("Queue %d not configured\n", i);
800                         return -ENOLINK;
801                 }
802
803         /* build up our prioritized array of qids */
804         /* We don't use qsort here, as if all/multiple entries have the same
805          * priority, the result is non-deterministic. From "man 3 qsort":
806          * "If two members compare as equal, their order in the sorted
807          * array is undefined."
808          */
809         uint32_t qidx = 0;
810         for (j = 0; j <= RTE_EVENT_DEV_PRIORITY_LOWEST; j++) {
811                 for (i = 0; i < sw->qid_count; i++) {
812                         if (sw->qids[i].priority == j) {
813                                 sw->qids_prioritized[qidx] = &sw->qids[i];
814                                 qidx++;
815                         }
816                 }
817         }
818
819         sw_init_qid_iqs(sw);
820
821         if (sw_xstats_init(sw) < 0)
822                 return -EINVAL;
823
824         rte_smp_wmb();
825         sw->started = 1;
826
827         return 0;
828 }
829
830 static void
831 sw_stop(struct rte_eventdev *dev)
832 {
833         struct sw_evdev *sw = sw_pmd_priv(dev);
834         int32_t runstate;
835
836         /* Stop the scheduler if it's running */
837         runstate = rte_service_runstate_get(sw->service_id);
838         if (runstate == 1)
839                 rte_service_runstate_set(sw->service_id, 0);
840
841         while (rte_service_may_be_active(sw->service_id))
842                 rte_pause();
843
844         /* Flush all events out of the device */
845         while (!(sw_qids_empty(sw) && sw_ports_empty(sw))) {
846                 sw_event_schedule(dev);
847                 sw_drain_ports(dev);
848                 sw_drain_queues(dev);
849         }
850
851         sw_clean_qid_iqs(dev);
852         sw_xstats_uninit(sw);
853         sw->started = 0;
854         rte_smp_wmb();
855
856         if (runstate == 1)
857                 rte_service_runstate_set(sw->service_id, 1);
858 }
859
860 static int
861 sw_close(struct rte_eventdev *dev)
862 {
863         struct sw_evdev *sw = sw_pmd_priv(dev);
864         uint32_t i;
865
866         for (i = 0; i < sw->qid_count; i++)
867                 sw_queue_release(dev, i);
868         sw->qid_count = 0;
869
870         for (i = 0; i < sw->port_count; i++)
871                 sw_port_release(&sw->ports[i]);
872         sw->port_count = 0;
873
874         memset(&sw->stats, 0, sizeof(sw->stats));
875         sw->sched_called = 0;
876         sw->sched_no_iq_enqueues = 0;
877         sw->sched_no_cq_enqueues = 0;
878         sw->sched_cq_qid_called = 0;
879
880         return 0;
881 }
882
883 static int
884 assign_numa_node(const char *key __rte_unused, const char *value, void *opaque)
885 {
886         int *socket_id = opaque;
887         *socket_id = atoi(value);
888         if (*socket_id >= RTE_MAX_NUMA_NODES)
889                 return -1;
890         return 0;
891 }
892
893 static int
894 set_sched_quanta(const char *key __rte_unused, const char *value, void *opaque)
895 {
896         int *quanta = opaque;
897         *quanta = atoi(value);
898         if (*quanta < 0 || *quanta >= 4096)
899                 return -1;
900         return 0;
901 }
902
903 static int
904 set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
905 {
906         int *credit = opaque;
907         *credit = atoi(value);
908         if (*credit < 0 || *credit >= 128)
909                 return -1;
910         return 0;
911 }
912
913
914 static int32_t sw_sched_service_func(void *args)
915 {
916         struct rte_eventdev *dev = args;
917         sw_event_schedule(dev);
918         return 0;
919 }
920
921 static int
922 sw_probe(struct rte_vdev_device *vdev)
923 {
924         static struct rte_eventdev_ops evdev_sw_ops = {
925                         .dev_configure = sw_dev_configure,
926                         .dev_infos_get = sw_info_get,
927                         .dev_close = sw_close,
928                         .dev_start = sw_start,
929                         .dev_stop = sw_stop,
930                         .dump = sw_dump,
931
932                         .queue_def_conf = sw_queue_def_conf,
933                         .queue_setup = sw_queue_setup,
934                         .queue_release = sw_queue_release,
935                         .port_def_conf = sw_port_def_conf,
936                         .port_setup = sw_port_setup,
937                         .port_release = sw_port_release,
938                         .port_link = sw_port_link,
939                         .port_unlink = sw_port_unlink,
940                         .port_unlinks_in_progress = sw_port_unlinks_in_progress,
941
942                         .eth_rx_adapter_caps_get = sw_eth_rx_adapter_caps_get,
943
944                         .timer_adapter_caps_get = sw_timer_adapter_caps_get,
945
946                         .crypto_adapter_caps_get = sw_crypto_adapter_caps_get,
947
948                         .xstats_get = sw_xstats_get,
949                         .xstats_get_names = sw_xstats_get_names,
950                         .xstats_get_by_name = sw_xstats_get_by_name,
951                         .xstats_reset = sw_xstats_reset,
952
953                         .dev_selftest = test_sw_eventdev,
954         };
955
956         static const char *const args[] = {
957                 NUMA_NODE_ARG,
958                 SCHED_QUANTA_ARG,
959                 CREDIT_QUANTA_ARG,
960                 NULL
961         };
962         const char *name;
963         const char *params;
964         struct rte_eventdev *dev;
965         struct sw_evdev *sw;
966         int socket_id = rte_socket_id();
967         int sched_quanta  = SW_DEFAULT_SCHED_QUANTA;
968         int credit_quanta = SW_DEFAULT_CREDIT_QUANTA;
969
970         name = rte_vdev_device_name(vdev);
971         params = rte_vdev_device_args(vdev);
972         if (params != NULL && params[0] != '\0') {
973                 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
974
975                 if (!kvlist) {
976                         SW_LOG_INFO(
977                                 "Ignoring unsupported parameters when creating device '%s'\n",
978                                 name);
979                 } else {
980                         int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
981                                         assign_numa_node, &socket_id);
982                         if (ret != 0) {
983                                 SW_LOG_ERR(
984                                         "%s: Error parsing numa node parameter",
985                                         name);
986                                 rte_kvargs_free(kvlist);
987                                 return ret;
988                         }
989
990                         ret = rte_kvargs_process(kvlist, SCHED_QUANTA_ARG,
991                                         set_sched_quanta, &sched_quanta);
992                         if (ret != 0) {
993                                 SW_LOG_ERR(
994                                         "%s: Error parsing sched quanta parameter",
995                                         name);
996                                 rte_kvargs_free(kvlist);
997                                 return ret;
998                         }
999
1000                         ret = rte_kvargs_process(kvlist, CREDIT_QUANTA_ARG,
1001                                         set_credit_quanta, &credit_quanta);
1002                         if (ret != 0) {
1003                                 SW_LOG_ERR(
1004                                         "%s: Error parsing credit quanta parameter",
1005                                         name);
1006                                 rte_kvargs_free(kvlist);
1007                                 return ret;
1008                         }
1009
1010                         rte_kvargs_free(kvlist);
1011                 }
1012         }
1013
1014         SW_LOG_INFO(
1015                         "Creating eventdev sw device %s, numa_node=%d, sched_quanta=%d, credit_quanta=%d\n",
1016                         name, socket_id, sched_quanta, credit_quanta);
1017
1018         dev = rte_event_pmd_vdev_init(name,
1019                         sizeof(struct sw_evdev), socket_id);
1020         if (dev == NULL) {
1021                 SW_LOG_ERR("eventdev vdev init() failed");
1022                 return -EFAULT;
1023         }
1024         dev->dev_ops = &evdev_sw_ops;
1025         dev->enqueue = sw_event_enqueue;
1026         dev->enqueue_burst = sw_event_enqueue_burst;
1027         dev->enqueue_new_burst = sw_event_enqueue_burst;
1028         dev->enqueue_forward_burst = sw_event_enqueue_burst;
1029         dev->dequeue = sw_event_dequeue;
1030         dev->dequeue_burst = sw_event_dequeue_burst;
1031
1032         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1033                 return 0;
1034
1035         sw = dev->data->dev_private;
1036         sw->data = dev->data;
1037
1038         /* copy values passed from vdev command line to instance */
1039         sw->credit_update_quanta = credit_quanta;
1040         sw->sched_quanta = sched_quanta;
1041
1042         /* register service with EAL */
1043         struct rte_service_spec service;
1044         memset(&service, 0, sizeof(struct rte_service_spec));
1045         snprintf(service.name, sizeof(service.name), "%s_service", name);
1046         snprintf(sw->service_name, sizeof(sw->service_name), "%s_service",
1047                         name);
1048         service.socket_id = socket_id;
1049         service.callback = sw_sched_service_func;
1050         service.callback_userdata = (void *)dev;
1051
1052         int32_t ret = rte_service_component_register(&service, &sw->service_id);
1053         if (ret) {
1054                 SW_LOG_ERR("service register() failed");
1055                 return -ENOEXEC;
1056         }
1057
1058         dev->data->service_inited = 1;
1059         dev->data->service_id = sw->service_id;
1060
1061         return 0;
1062 }
1063
1064 static int
1065 sw_remove(struct rte_vdev_device *vdev)
1066 {
1067         const char *name;
1068
1069         name = rte_vdev_device_name(vdev);
1070         if (name == NULL)
1071                 return -EINVAL;
1072
1073         SW_LOG_INFO("Closing eventdev sw device %s\n", name);
1074
1075         return rte_event_pmd_vdev_uninit(name);
1076 }
1077
1078 static struct rte_vdev_driver evdev_sw_pmd_drv = {
1079         .probe = sw_probe,
1080         .remove = sw_remove
1081 };
1082
1083 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SW_PMD, evdev_sw_pmd_drv);
1084 RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "=<int> "
1085                 SCHED_QUANTA_ARG "=<int>" CREDIT_QUANTA_ARG "=<int>");
1086
1087 /* declared extern in header, for access from other .c files */
1088 int eventdev_sw_log_level;
1089
1090 RTE_INIT(evdev_sw_init_log)
1091 {
1092         eventdev_sw_log_level = rte_log_register("pmd.event.sw");
1093         if (eventdev_sw_log_level >= 0)
1094                 rte_log_set_level(eventdev_sw_log_level, RTE_LOG_NOTICE);
1095 }