net/virtio: fix incorrect cast of void *
[dpdk.git] / test / test / test_eventdev_sw.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2016-2017 Intel Corporation. All rights reserved.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include <stdio.h>
35 #include <string.h>
36 #include <stdint.h>
37 #include <errno.h>
38 #include <unistd.h>
39 #include <sys/queue.h>
40
41 #include <rte_memory.h>
42 #include <rte_launch.h>
43 #include <rte_eal.h>
44 #include <rte_per_lcore.h>
45 #include <rte_lcore.h>
46 #include <rte_debug.h>
47 #include <rte_ethdev.h>
48 #include <rte_cycles.h>
49 #include <rte_eventdev.h>
50 #include <rte_pause.h>
51 #include <rte_service.h>
52 #include <rte_service_component.h>
53 #include <rte_bus_vdev.h>
54
55 #include "test.h"
56
57 #define MAX_PORTS 16
58 #define MAX_QIDS 16
59 #define NUM_PACKETS (1<<18)
60
61 static int evdev;
62
63 struct test {
64         struct rte_mempool *mbuf_pool;
65         uint8_t port[MAX_PORTS];
66         uint8_t qid[MAX_QIDS];
67         int nb_qids;
68         uint32_t service_id;
69 };
70
71 static struct rte_event release_ev;
72
73 static inline struct rte_mbuf *
74 rte_gen_arp(int portid, struct rte_mempool *mp)
75 {
76         /*
77          * len = 14 + 46
78          * ARP, Request who-has 10.0.0.1 tell 10.0.0.2, length 46
79          */
80         static const uint8_t arp_request[] = {
81                 /*0x0000:*/ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xec, 0xa8,
82                 0x6b, 0xfd, 0x02, 0x29, 0x08, 0x06, 0x00, 0x01,
83                 /*0x0010:*/ 0x08, 0x00, 0x06, 0x04, 0x00, 0x01, 0xec, 0xa8,
84                 0x6b, 0xfd, 0x02, 0x29, 0x0a, 0x00, 0x00, 0x01,
85                 /*0x0020:*/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00,
86                 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
87                 /*0x0030:*/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
88                 0x00, 0x00, 0x00, 0x00
89         };
90         struct rte_mbuf *m;
91         int pkt_len = sizeof(arp_request) - 1;
92
93         m = rte_pktmbuf_alloc(mp);
94         if (!m)
95                 return 0;
96
97         memcpy((void *)((uintptr_t)m->buf_addr + m->data_off),
98                 arp_request, pkt_len);
99         rte_pktmbuf_pkt_len(m) = pkt_len;
100         rte_pktmbuf_data_len(m) = pkt_len;
101
102         RTE_SET_USED(portid);
103
104         return m;
105 }
106
107 static void
108 xstats_print(void)
109 {
110         const uint32_t XSTATS_MAX = 1024;
111         uint32_t i;
112         uint32_t ids[XSTATS_MAX];
113         uint64_t values[XSTATS_MAX];
114         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
115
116         for (i = 0; i < XSTATS_MAX; i++)
117                 ids[i] = i;
118
119         /* Device names / values */
120         int ret = rte_event_dev_xstats_names_get(evdev,
121                                         RTE_EVENT_DEV_XSTATS_DEVICE, 0,
122                                         xstats_names, ids, XSTATS_MAX);
123         if (ret < 0) {
124                 printf("%d: xstats names get() returned error\n",
125                         __LINE__);
126                 return;
127         }
128         ret = rte_event_dev_xstats_get(evdev,
129                                         RTE_EVENT_DEV_XSTATS_DEVICE,
130                                         0, ids, values, ret);
131         if (ret > (signed int)XSTATS_MAX)
132                 printf("%s %d: more xstats available than space\n",
133                                 __func__, __LINE__);
134         for (i = 0; (signed int)i < ret; i++) {
135                 printf("%d : %s : %"PRIu64"\n",
136                                 i, xstats_names[i].name, values[i]);
137         }
138
139         /* Port names / values */
140         ret = rte_event_dev_xstats_names_get(evdev,
141                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
142                                         xstats_names, ids, XSTATS_MAX);
143         ret = rte_event_dev_xstats_get(evdev,
144                                         RTE_EVENT_DEV_XSTATS_PORT, 1,
145                                         ids, values, ret);
146         if (ret > (signed int)XSTATS_MAX)
147                 printf("%s %d: more xstats available than space\n",
148                                 __func__, __LINE__);
149         for (i = 0; (signed int)i < ret; i++) {
150                 printf("%d : %s : %"PRIu64"\n",
151                                 i, xstats_names[i].name, values[i]);
152         }
153
154         /* Queue names / values */
155         ret = rte_event_dev_xstats_names_get(evdev,
156                                         RTE_EVENT_DEV_XSTATS_QUEUE, 0,
157                                         xstats_names, ids, XSTATS_MAX);
158         ret = rte_event_dev_xstats_get(evdev,
159                                         RTE_EVENT_DEV_XSTATS_QUEUE,
160                                         1, ids, values, ret);
161         if (ret > (signed int)XSTATS_MAX)
162                 printf("%s %d: more xstats available than space\n",
163                                 __func__, __LINE__);
164         for (i = 0; (signed int)i < ret; i++) {
165                 printf("%d : %s : %"PRIu64"\n",
166                                 i, xstats_names[i].name, values[i]);
167         }
168 }
169
170 /* initialization and config */
171 static inline int
172 init(struct test *t, int nb_queues, int nb_ports)
173 {
174         struct rte_event_dev_config config = {
175                         .nb_event_queues = nb_queues,
176                         .nb_event_ports = nb_ports,
177                         .nb_event_queue_flows = 1024,
178                         .nb_events_limit = 4096,
179                         .nb_event_port_dequeue_depth = 128,
180                         .nb_event_port_enqueue_depth = 128,
181         };
182         int ret;
183
184         void *temp = t->mbuf_pool; /* save and restore mbuf pool */
185
186         memset(t, 0, sizeof(*t));
187         t->mbuf_pool = temp;
188
189         ret = rte_event_dev_configure(evdev, &config);
190         if (ret < 0)
191                 printf("%d: Error configuring device\n", __LINE__);
192         return ret;
193 };
194
195 static inline int
196 create_ports(struct test *t, int num_ports)
197 {
198         int i;
199         static const struct rte_event_port_conf conf = {
200                         .new_event_threshold = 1024,
201                         .dequeue_depth = 32,
202                         .enqueue_depth = 64,
203         };
204         if (num_ports > MAX_PORTS)
205                 return -1;
206
207         for (i = 0; i < num_ports; i++) {
208                 if (rte_event_port_setup(evdev, i, &conf) < 0) {
209                         printf("Error setting up port %d\n", i);
210                         return -1;
211                 }
212                 t->port[i] = i;
213         }
214
215         return 0;
216 }
217
218 static inline int
219 create_lb_qids(struct test *t, int num_qids, uint32_t flags)
220 {
221         int i;
222
223         /* Q creation */
224         const struct rte_event_queue_conf conf = {
225                         .schedule_type = flags,
226                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
227                         .nb_atomic_flows = 1024,
228                         .nb_atomic_order_sequences = 1024,
229         };
230
231         for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
232                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
233                         printf("%d: error creating qid %d\n", __LINE__, i);
234                         return -1;
235                 }
236                 t->qid[i] = i;
237         }
238         t->nb_qids += num_qids;
239         if (t->nb_qids > MAX_QIDS)
240                 return -1;
241
242         return 0;
243 }
244
245 static inline int
246 create_atomic_qids(struct test *t, int num_qids)
247 {
248         return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
249 }
250
251 static inline int
252 create_ordered_qids(struct test *t, int num_qids)
253 {
254         return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ORDERED);
255 }
256
257
258 static inline int
259 create_unordered_qids(struct test *t, int num_qids)
260 {
261         return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_PARALLEL);
262 }
263
264 static inline int
265 create_directed_qids(struct test *t, int num_qids, const uint8_t ports[])
266 {
267         int i;
268
269         /* Q creation */
270         static const struct rte_event_queue_conf conf = {
271                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
272                         .event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK,
273         };
274
275         for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
276                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
277                         printf("%d: error creating qid %d\n", __LINE__, i);
278                         return -1;
279                 }
280                 t->qid[i] = i;
281
282                 if (rte_event_port_link(evdev, ports[i - t->nb_qids],
283                                 &t->qid[i], NULL, 1) != 1) {
284                         printf("%d: error creating link for qid %d\n",
285                                         __LINE__, i);
286                         return -1;
287                 }
288         }
289         t->nb_qids += num_qids;
290         if (t->nb_qids > MAX_QIDS)
291                 return -1;
292
293         return 0;
294 }
295
296 /* destruction */
297 static inline int
298 cleanup(struct test *t __rte_unused)
299 {
300         rte_event_dev_stop(evdev);
301         rte_event_dev_close(evdev);
302         return 0;
303 };
304
305 struct test_event_dev_stats {
306         uint64_t rx_pkts;       /**< Total packets received */
307         uint64_t rx_dropped;    /**< Total packets dropped (Eg Invalid QID) */
308         uint64_t tx_pkts;       /**< Total packets transmitted */
309
310         /** Packets received on this port */
311         uint64_t port_rx_pkts[MAX_PORTS];
312         /** Packets dropped on this port */
313         uint64_t port_rx_dropped[MAX_PORTS];
314         /** Packets inflight on this port */
315         uint64_t port_inflight[MAX_PORTS];
316         /** Packets transmitted on this port */
317         uint64_t port_tx_pkts[MAX_PORTS];
318         /** Packets received on this qid */
319         uint64_t qid_rx_pkts[MAX_QIDS];
320         /** Packets dropped on this qid */
321         uint64_t qid_rx_dropped[MAX_QIDS];
322         /** Packets transmitted on this qid */
323         uint64_t qid_tx_pkts[MAX_QIDS];
324 };
325
326 static inline int
327 test_event_dev_stats_get(int dev_id, struct test_event_dev_stats *stats)
328 {
329         static uint32_t i;
330         static uint32_t total_ids[3]; /* rx, tx and drop */
331         static uint32_t port_rx_pkts_ids[MAX_PORTS];
332         static uint32_t port_rx_dropped_ids[MAX_PORTS];
333         static uint32_t port_inflight_ids[MAX_PORTS];
334         static uint32_t port_tx_pkts_ids[MAX_PORTS];
335         static uint32_t qid_rx_pkts_ids[MAX_QIDS];
336         static uint32_t qid_rx_dropped_ids[MAX_QIDS];
337         static uint32_t qid_tx_pkts_ids[MAX_QIDS];
338
339
340         stats->rx_pkts = rte_event_dev_xstats_by_name_get(dev_id,
341                         "dev_rx", &total_ids[0]);
342         stats->rx_dropped = rte_event_dev_xstats_by_name_get(dev_id,
343                         "dev_drop", &total_ids[1]);
344         stats->tx_pkts = rte_event_dev_xstats_by_name_get(dev_id,
345                         "dev_tx", &total_ids[2]);
346         for (i = 0; i < MAX_PORTS; i++) {
347                 char name[32];
348                 snprintf(name, sizeof(name), "port_%u_rx", i);
349                 stats->port_rx_pkts[i] = rte_event_dev_xstats_by_name_get(
350                                 dev_id, name, &port_rx_pkts_ids[i]);
351                 snprintf(name, sizeof(name), "port_%u_drop", i);
352                 stats->port_rx_dropped[i] = rte_event_dev_xstats_by_name_get(
353                                 dev_id, name, &port_rx_dropped_ids[i]);
354                 snprintf(name, sizeof(name), "port_%u_inflight", i);
355                 stats->port_inflight[i] = rte_event_dev_xstats_by_name_get(
356                                 dev_id, name, &port_inflight_ids[i]);
357                 snprintf(name, sizeof(name), "port_%u_tx", i);
358                 stats->port_tx_pkts[i] = rte_event_dev_xstats_by_name_get(
359                                 dev_id, name, &port_tx_pkts_ids[i]);
360         }
361         for (i = 0; i < MAX_QIDS; i++) {
362                 char name[32];
363                 snprintf(name, sizeof(name), "qid_%u_rx", i);
364                 stats->qid_rx_pkts[i] = rte_event_dev_xstats_by_name_get(
365                                 dev_id, name, &qid_rx_pkts_ids[i]);
366                 snprintf(name, sizeof(name), "qid_%u_drop", i);
367                 stats->qid_rx_dropped[i] = rte_event_dev_xstats_by_name_get(
368                                 dev_id, name, &qid_rx_dropped_ids[i]);
369                 snprintf(name, sizeof(name), "qid_%u_tx", i);
370                 stats->qid_tx_pkts[i] = rte_event_dev_xstats_by_name_get(
371                                 dev_id, name, &qid_tx_pkts_ids[i]);
372         }
373
374         return 0;
375 }
376
377 /* run_prio_packet_test
378  * This performs a basic packet priority check on the test instance passed in.
379  * It is factored out of the main priority tests as the same tests must be
380  * performed to ensure prioritization of each type of QID.
381  *
382  * Requirements:
383  *  - An initialized test structure, including mempool
384  *  - t->port[0] is initialized for both Enq / Deq of packets to the QID
385  *  - t->qid[0] is the QID to be tested
386  *  - if LB QID, the CQ must be mapped to the QID.
387  */
388 static int
389 run_prio_packet_test(struct test *t)
390 {
391         int err;
392         const uint32_t MAGIC_SEQN[] = {4711, 1234};
393         const uint32_t PRIORITY[] = {
394                 RTE_EVENT_DEV_PRIORITY_NORMAL,
395                 RTE_EVENT_DEV_PRIORITY_HIGHEST
396         };
397         unsigned int i;
398         for (i = 0; i < RTE_DIM(MAGIC_SEQN); i++) {
399                 /* generate pkt and enqueue */
400                 struct rte_event ev;
401                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
402                 if (!arp) {
403                         printf("%d: gen of pkt failed\n", __LINE__);
404                         return -1;
405                 }
406                 arp->seqn = MAGIC_SEQN[i];
407
408                 ev = (struct rte_event){
409                         .priority = PRIORITY[i],
410                         .op = RTE_EVENT_OP_NEW,
411                         .queue_id = t->qid[0],
412                         .mbuf = arp
413                 };
414                 err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
415                 if (err < 0) {
416                         printf("%d: error failed to enqueue\n", __LINE__);
417                         return -1;
418                 }
419         }
420
421         rte_service_run_iter_on_app_lcore(t->service_id, 1);
422
423         struct test_event_dev_stats stats;
424         err = test_event_dev_stats_get(evdev, &stats);
425         if (err) {
426                 printf("%d: error failed to get stats\n", __LINE__);
427                 return -1;
428         }
429
430         if (stats.port_rx_pkts[t->port[0]] != 2) {
431                 printf("%d: error stats incorrect for directed port\n",
432                                 __LINE__);
433                 rte_event_dev_dump(evdev, stdout);
434                 return -1;
435         }
436
437         struct rte_event ev, ev2;
438         uint32_t deq_pkts;
439         deq_pkts = rte_event_dequeue_burst(evdev, t->port[0], &ev, 1, 0);
440         if (deq_pkts != 1) {
441                 printf("%d: error failed to deq\n", __LINE__);
442                 rte_event_dev_dump(evdev, stdout);
443                 return -1;
444         }
445         if (ev.mbuf->seqn != MAGIC_SEQN[1]) {
446                 printf("%d: first packet out not highest priority\n",
447                                 __LINE__);
448                 rte_event_dev_dump(evdev, stdout);
449                 return -1;
450         }
451         rte_pktmbuf_free(ev.mbuf);
452
453         deq_pkts = rte_event_dequeue_burst(evdev, t->port[0], &ev2, 1, 0);
454         if (deq_pkts != 1) {
455                 printf("%d: error failed to deq\n", __LINE__);
456                 rte_event_dev_dump(evdev, stdout);
457                 return -1;
458         }
459         if (ev2.mbuf->seqn != MAGIC_SEQN[0]) {
460                 printf("%d: second packet out not lower priority\n",
461                                 __LINE__);
462                 rte_event_dev_dump(evdev, stdout);
463                 return -1;
464         }
465         rte_pktmbuf_free(ev2.mbuf);
466
467         cleanup(t);
468         return 0;
469 }
470
471 static int
472 test_single_directed_packet(struct test *t)
473 {
474         const int rx_enq = 0;
475         const int wrk_enq = 2;
476         int err;
477
478         /* Create instance with 3 directed QIDs going to 3 ports */
479         if (init(t, 3, 3) < 0 ||
480                         create_ports(t, 3) < 0 ||
481                         create_directed_qids(t, 3, t->port) < 0)
482                 return -1;
483
484         if (rte_event_dev_start(evdev) < 0) {
485                 printf("%d: Error with start call\n", __LINE__);
486                 return -1;
487         }
488
489         /************** FORWARD ****************/
490         struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
491         struct rte_event ev = {
492                         .op = RTE_EVENT_OP_NEW,
493                         .queue_id = wrk_enq,
494                         .mbuf = arp,
495         };
496
497         if (!arp) {
498                 printf("%d: gen of pkt failed\n", __LINE__);
499                 return -1;
500         }
501
502         const uint32_t MAGIC_SEQN = 4711;
503         arp->seqn = MAGIC_SEQN;
504
505         /* generate pkt and enqueue */
506         err = rte_event_enqueue_burst(evdev, rx_enq, &ev, 1);
507         if (err < 0) {
508                 printf("%d: error failed to enqueue\n", __LINE__);
509                 return -1;
510         }
511
512         /* Run schedule() as dir packets may need to be re-ordered */
513         rte_service_run_iter_on_app_lcore(t->service_id, 1);
514
515         struct test_event_dev_stats stats;
516         err = test_event_dev_stats_get(evdev, &stats);
517         if (err) {
518                 printf("%d: error failed to get stats\n", __LINE__);
519                 return -1;
520         }
521
522         if (stats.port_rx_pkts[rx_enq] != 1) {
523                 printf("%d: error stats incorrect for directed port\n",
524                                 __LINE__);
525                 return -1;
526         }
527
528         uint32_t deq_pkts;
529         deq_pkts = rte_event_dequeue_burst(evdev, wrk_enq, &ev, 1, 0);
530         if (deq_pkts != 1) {
531                 printf("%d: error failed to deq\n", __LINE__);
532                 return -1;
533         }
534
535         err = test_event_dev_stats_get(evdev, &stats);
536         if (stats.port_rx_pkts[wrk_enq] != 0 &&
537                         stats.port_rx_pkts[wrk_enq] != 1) {
538                 printf("%d: error directed stats post-dequeue\n", __LINE__);
539                 return -1;
540         }
541
542         if (ev.mbuf->seqn != MAGIC_SEQN) {
543                 printf("%d: error magic sequence number not dequeued\n",
544                                 __LINE__);
545                 return -1;
546         }
547
548         rte_pktmbuf_free(ev.mbuf);
549         cleanup(t);
550         return 0;
551 }
552
553 static int
554 test_directed_forward_credits(struct test *t)
555 {
556         uint32_t i;
557         int32_t err;
558
559         if (init(t, 1, 1) < 0 ||
560                         create_ports(t, 1) < 0 ||
561                         create_directed_qids(t, 1, t->port) < 0)
562                 return -1;
563
564         if (rte_event_dev_start(evdev) < 0) {
565                 printf("%d: Error with start call\n", __LINE__);
566                 return -1;
567         }
568
569         struct rte_event ev = {
570                         .op = RTE_EVENT_OP_NEW,
571                         .queue_id = 0,
572         };
573
574         for (i = 0; i < 1000; i++) {
575                 err = rte_event_enqueue_burst(evdev, 0, &ev, 1);
576                 if (err < 0) {
577                         printf("%d: error failed to enqueue\n", __LINE__);
578                         return -1;
579                 }
580                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
581
582                 uint32_t deq_pkts;
583                 deq_pkts = rte_event_dequeue_burst(evdev, 0, &ev, 1, 0);
584                 if (deq_pkts != 1) {
585                         printf("%d: error failed to deq\n", __LINE__);
586                         return -1;
587                 }
588
589                 /* re-write event to be a forward, and continue looping it */
590                 ev.op = RTE_EVENT_OP_FORWARD;
591         }
592
593         cleanup(t);
594         return 0;
595 }
596
597
598 static int
599 test_priority_directed(struct test *t)
600 {
601         if (init(t, 1, 1) < 0 ||
602                         create_ports(t, 1) < 0 ||
603                         create_directed_qids(t, 1, t->port) < 0) {
604                 printf("%d: Error initializing device\n", __LINE__);
605                 return -1;
606         }
607
608         if (rte_event_dev_start(evdev) < 0) {
609                 printf("%d: Error with start call\n", __LINE__);
610                 return -1;
611         }
612
613         return run_prio_packet_test(t);
614 }
615
616 static int
617 test_priority_atomic(struct test *t)
618 {
619         if (init(t, 1, 1) < 0 ||
620                         create_ports(t, 1) < 0 ||
621                         create_atomic_qids(t, 1) < 0) {
622                 printf("%d: Error initializing device\n", __LINE__);
623                 return -1;
624         }
625
626         /* map the QID */
627         if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) {
628                 printf("%d: error mapping qid to port\n", __LINE__);
629                 return -1;
630         }
631         if (rte_event_dev_start(evdev) < 0) {
632                 printf("%d: Error with start call\n", __LINE__);
633                 return -1;
634         }
635
636         return run_prio_packet_test(t);
637 }
638
639 static int
640 test_priority_ordered(struct test *t)
641 {
642         if (init(t, 1, 1) < 0 ||
643                         create_ports(t, 1) < 0 ||
644                         create_ordered_qids(t, 1) < 0) {
645                 printf("%d: Error initializing device\n", __LINE__);
646                 return -1;
647         }
648
649         /* map the QID */
650         if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) {
651                 printf("%d: error mapping qid to port\n", __LINE__);
652                 return -1;
653         }
654         if (rte_event_dev_start(evdev) < 0) {
655                 printf("%d: Error with start call\n", __LINE__);
656                 return -1;
657         }
658
659         return run_prio_packet_test(t);
660 }
661
662 static int
663 test_priority_unordered(struct test *t)
664 {
665         if (init(t, 1, 1) < 0 ||
666                         create_ports(t, 1) < 0 ||
667                         create_unordered_qids(t, 1) < 0) {
668                 printf("%d: Error initializing device\n", __LINE__);
669                 return -1;
670         }
671
672         /* map the QID */
673         if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) {
674                 printf("%d: error mapping qid to port\n", __LINE__);
675                 return -1;
676         }
677         if (rte_event_dev_start(evdev) < 0) {
678                 printf("%d: Error with start call\n", __LINE__);
679                 return -1;
680         }
681
682         return run_prio_packet_test(t);
683 }
684
685 static int
686 burst_packets(struct test *t)
687 {
688         /************** CONFIG ****************/
689         uint32_t i;
690         int err;
691         int ret;
692
693         /* Create instance with 2 ports and 2 queues */
694         if (init(t, 2, 2) < 0 ||
695                         create_ports(t, 2) < 0 ||
696                         create_atomic_qids(t, 2) < 0) {
697                 printf("%d: Error initializing device\n", __LINE__);
698                 return -1;
699         }
700
701         /* CQ mapping to QID */
702         ret = rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1);
703         if (ret != 1) {
704                 printf("%d: error mapping lb qid0\n", __LINE__);
705                 return -1;
706         }
707         ret = rte_event_port_link(evdev, t->port[1], &t->qid[1], NULL, 1);
708         if (ret != 1) {
709                 printf("%d: error mapping lb qid1\n", __LINE__);
710                 return -1;
711         }
712
713         if (rte_event_dev_start(evdev) < 0) {
714                 printf("%d: Error with start call\n", __LINE__);
715                 return -1;
716         }
717
718         /************** FORWARD ****************/
719         const uint32_t rx_port = 0;
720         const uint32_t NUM_PKTS = 2;
721
722         for (i = 0; i < NUM_PKTS; i++) {
723                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
724                 if (!arp) {
725                         printf("%d: error generating pkt\n", __LINE__);
726                         return -1;
727                 }
728
729                 struct rte_event ev = {
730                                 .op = RTE_EVENT_OP_NEW,
731                                 .queue_id = i % 2,
732                                 .flow_id = i % 3,
733                                 .mbuf = arp,
734                 };
735                 /* generate pkt and enqueue */
736                 err = rte_event_enqueue_burst(evdev, t->port[rx_port], &ev, 1);
737                 if (err < 0) {
738                         printf("%d: Failed to enqueue\n", __LINE__);
739                         return -1;
740                 }
741         }
742         rte_service_run_iter_on_app_lcore(t->service_id, 1);
743
744         /* Check stats for all NUM_PKTS arrived to sched core */
745         struct test_event_dev_stats stats;
746
747         err = test_event_dev_stats_get(evdev, &stats);
748         if (err) {
749                 printf("%d: failed to get stats\n", __LINE__);
750                 return -1;
751         }
752         if (stats.rx_pkts != NUM_PKTS || stats.tx_pkts != NUM_PKTS) {
753                 printf("%d: Sched core didn't receive all %d pkts\n",
754                                 __LINE__, NUM_PKTS);
755                 rte_event_dev_dump(evdev, stdout);
756                 return -1;
757         }
758
759         uint32_t deq_pkts;
760         int p;
761
762         deq_pkts = 0;
763         /******** DEQ QID 1 *******/
764         do {
765                 struct rte_event ev;
766                 p = rte_event_dequeue_burst(evdev, t->port[0], &ev, 1, 0);
767                 deq_pkts += p;
768                 rte_pktmbuf_free(ev.mbuf);
769         } while (p);
770
771         if (deq_pkts != NUM_PKTS/2) {
772                 printf("%d: Half of NUM_PKTS didn't arrive at port 1\n",
773                                 __LINE__);
774                 return -1;
775         }
776
777         /******** DEQ QID 2 *******/
778         deq_pkts = 0;
779         do {
780                 struct rte_event ev;
781                 p = rte_event_dequeue_burst(evdev, t->port[1], &ev, 1, 0);
782                 deq_pkts += p;
783                 rte_pktmbuf_free(ev.mbuf);
784         } while (p);
785         if (deq_pkts != NUM_PKTS/2) {
786                 printf("%d: Half of NUM_PKTS didn't arrive at port 2\n",
787                                 __LINE__);
788                 return -1;
789         }
790
791         cleanup(t);
792         return 0;
793 }
794
795 static int
796 abuse_inflights(struct test *t)
797 {
798         const int rx_enq = 0;
799         const int wrk_enq = 2;
800         int err;
801
802         /* Create instance with 4 ports */
803         if (init(t, 1, 4) < 0 ||
804                         create_ports(t, 4) < 0 ||
805                         create_atomic_qids(t, 1) < 0) {
806                 printf("%d: Error initializing device\n", __LINE__);
807                 return -1;
808         }
809
810         /* CQ mapping to QID */
811         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
812         if (err != 1) {
813                 printf("%d: error mapping lb qid\n", __LINE__);
814                 cleanup(t);
815                 return -1;
816         }
817
818         if (rte_event_dev_start(evdev) < 0) {
819                 printf("%d: Error with start call\n", __LINE__);
820                 return -1;
821         }
822
823         /* Enqueue op only */
824         err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &release_ev, 1);
825         if (err < 0) {
826                 printf("%d: Failed to enqueue\n", __LINE__);
827                 return -1;
828         }
829
830         /* schedule */
831         rte_service_run_iter_on_app_lcore(t->service_id, 1);
832
833         struct test_event_dev_stats stats;
834
835         err = test_event_dev_stats_get(evdev, &stats);
836         if (err) {
837                 printf("%d: failed to get stats\n", __LINE__);
838                 return -1;
839         }
840
841         if (stats.rx_pkts != 0 ||
842                         stats.tx_pkts != 0 ||
843                         stats.port_inflight[wrk_enq] != 0) {
844                 printf("%d: Sched core didn't handle pkt as expected\n",
845                                 __LINE__);
846                 return -1;
847         }
848
849         cleanup(t);
850         return 0;
851 }
852
853 static int
854 xstats_tests(struct test *t)
855 {
856         const int wrk_enq = 2;
857         int err;
858
859         /* Create instance with 4 ports */
860         if (init(t, 1, 4) < 0 ||
861                         create_ports(t, 4) < 0 ||
862                         create_atomic_qids(t, 1) < 0) {
863                 printf("%d: Error initializing device\n", __LINE__);
864                 return -1;
865         }
866
867         /* CQ mapping to QID */
868         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
869         if (err != 1) {
870                 printf("%d: error mapping lb qid\n", __LINE__);
871                 cleanup(t);
872                 return -1;
873         }
874
875         if (rte_event_dev_start(evdev) < 0) {
876                 printf("%d: Error with start call\n", __LINE__);
877                 return -1;
878         }
879
880         const uint32_t XSTATS_MAX = 1024;
881
882         uint32_t i;
883         uint32_t ids[XSTATS_MAX];
884         uint64_t values[XSTATS_MAX];
885         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
886
887         for (i = 0; i < XSTATS_MAX; i++)
888                 ids[i] = i;
889
890         /* Device names / values */
891         int ret = rte_event_dev_xstats_names_get(evdev,
892                                         RTE_EVENT_DEV_XSTATS_DEVICE,
893                                         0, xstats_names, ids, XSTATS_MAX);
894         if (ret != 6) {
895                 printf("%d: expected 6 stats, got return %d\n", __LINE__, ret);
896                 return -1;
897         }
898         ret = rte_event_dev_xstats_get(evdev,
899                                         RTE_EVENT_DEV_XSTATS_DEVICE,
900                                         0, ids, values, ret);
901         if (ret != 6) {
902                 printf("%d: expected 6 stats, got return %d\n", __LINE__, ret);
903                 return -1;
904         }
905
906         /* Port names / values */
907         ret = rte_event_dev_xstats_names_get(evdev,
908                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
909                                         xstats_names, ids, XSTATS_MAX);
910         if (ret != 21) {
911                 printf("%d: expected 21 stats, got return %d\n", __LINE__, ret);
912                 return -1;
913         }
914         ret = rte_event_dev_xstats_get(evdev,
915                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
916                                         ids, values, ret);
917         if (ret != 21) {
918                 printf("%d: expected 21 stats, got return %d\n", __LINE__, ret);
919                 return -1;
920         }
921
922         /* Queue names / values */
923         ret = rte_event_dev_xstats_names_get(evdev,
924                                         RTE_EVENT_DEV_XSTATS_QUEUE,
925                                         0, xstats_names, ids, XSTATS_MAX);
926         if (ret != 17) {
927                 printf("%d: expected 17 stats, got return %d\n", __LINE__, ret);
928                 return -1;
929         }
930
931         /* NEGATIVE TEST: with wrong queue passed, 0 stats should be returned */
932         ret = rte_event_dev_xstats_get(evdev,
933                                         RTE_EVENT_DEV_XSTATS_QUEUE,
934                                         1, ids, values, ret);
935         if (ret != -EINVAL) {
936                 printf("%d: expected 0 stats, got return %d\n", __LINE__, ret);
937                 return -1;
938         }
939
940         ret = rte_event_dev_xstats_get(evdev,
941                                         RTE_EVENT_DEV_XSTATS_QUEUE,
942                                         0, ids, values, ret);
943         if (ret != 17) {
944                 printf("%d: expected 17 stats, got return %d\n", __LINE__, ret);
945                 return -1;
946         }
947
948         /* enqueue packets to check values */
949         for (i = 0; i < 3; i++) {
950                 struct rte_event ev;
951                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
952                 if (!arp) {
953                         printf("%d: gen of pkt failed\n", __LINE__);
954                         return -1;
955                 }
956                 ev.queue_id = t->qid[i];
957                 ev.op = RTE_EVENT_OP_NEW;
958                 ev.mbuf = arp;
959                 ev.flow_id = 7;
960                 arp->seqn = i;
961
962                 int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
963                 if (err != 1) {
964                         printf("%d: Failed to enqueue\n", __LINE__);
965                         return -1;
966                 }
967         }
968
969         rte_service_run_iter_on_app_lcore(t->service_id, 1);
970
971         /* Device names / values */
972         int num_stats = rte_event_dev_xstats_names_get(evdev,
973                                         RTE_EVENT_DEV_XSTATS_DEVICE, 0,
974                                         xstats_names, ids, XSTATS_MAX);
975         if (num_stats < 0)
976                 goto fail;
977         ret = rte_event_dev_xstats_get(evdev,
978                                         RTE_EVENT_DEV_XSTATS_DEVICE,
979                                         0, ids, values, num_stats);
980         static const uint64_t expected[] = {3, 3, 0, 1, 0, 0};
981         for (i = 0; (signed int)i < ret; i++) {
982                 if (expected[i] != values[i]) {
983                         printf(
984                                 "%d Error xstat %d (id %d) %s : %"PRIu64
985                                 ", expect %"PRIu64"\n",
986                                 __LINE__, i, ids[i], xstats_names[i].name,
987                                 values[i], expected[i]);
988                         goto fail;
989                 }
990         }
991
992         ret = rte_event_dev_xstats_reset(evdev, RTE_EVENT_DEV_XSTATS_DEVICE,
993                                         0, NULL, 0);
994
995         /* ensure reset statistics are zero-ed */
996         static const uint64_t expected_zero[] = {0, 0, 0, 0, 0, 0};
997         ret = rte_event_dev_xstats_get(evdev,
998                                         RTE_EVENT_DEV_XSTATS_DEVICE,
999                                         0, ids, values, num_stats);
1000         for (i = 0; (signed int)i < ret; i++) {
1001                 if (expected_zero[i] != values[i]) {
1002                         printf(
1003                                 "%d Error, xstat %d (id %d) %s : %"PRIu64
1004                                 ", expect %"PRIu64"\n",
1005                                 __LINE__, i, ids[i], xstats_names[i].name,
1006                                 values[i], expected_zero[i]);
1007                         goto fail;
1008                 }
1009         }
1010
1011         /* port reset checks */
1012         num_stats = rte_event_dev_xstats_names_get(evdev,
1013                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
1014                                         xstats_names, ids, XSTATS_MAX);
1015         if (num_stats < 0)
1016                 goto fail;
1017         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_PORT,
1018                                         0, ids, values, num_stats);
1019
1020         static const uint64_t port_expected[] = {
1021                 3 /* rx */,
1022                 0 /* tx */,
1023                 0 /* drop */,
1024                 0 /* inflights */,
1025                 0 /* avg pkt cycles */,
1026                 29 /* credits */,
1027                 0 /* rx ring used */,
1028                 4096 /* rx ring free */,
1029                 0 /* cq ring used */,
1030                 32 /* cq ring free */,
1031                 0 /* dequeue calls */,
1032                 /* 10 dequeue burst buckets */
1033                 0, 0, 0, 0, 0,
1034                 0, 0, 0, 0, 0,
1035         };
1036         if (ret != RTE_DIM(port_expected)) {
1037                 printf(
1038                         "%s %d: wrong number of port stats (%d), expected %zu\n",
1039                         __func__, __LINE__, ret, RTE_DIM(port_expected));
1040         }
1041
1042         for (i = 0; (signed int)i < ret; i++) {
1043                 if (port_expected[i] != values[i]) {
1044                         printf(
1045                                 "%s : %d: Error stat %s is %"PRIu64
1046                                 ", expected %"PRIu64"\n",
1047                                 __func__, __LINE__, xstats_names[i].name,
1048                                 values[i], port_expected[i]);
1049                         goto fail;
1050                 }
1051         }
1052
1053         ret = rte_event_dev_xstats_reset(evdev, RTE_EVENT_DEV_XSTATS_PORT,
1054                                         0, NULL, 0);
1055
1056         /* ensure reset statistics are zero-ed */
1057         static const uint64_t port_expected_zero[] = {
1058                 0 /* rx */,
1059                 0 /* tx */,
1060                 0 /* drop */,
1061                 0 /* inflights */,
1062                 0 /* avg pkt cycles */,
1063                 29 /* credits */,
1064                 0 /* rx ring used */,
1065                 4096 /* rx ring free */,
1066                 0 /* cq ring used */,
1067                 32 /* cq ring free */,
1068                 0 /* dequeue calls */,
1069                 /* 10 dequeue burst buckets */
1070                 0, 0, 0, 0, 0,
1071                 0, 0, 0, 0, 0,
1072         };
1073         ret = rte_event_dev_xstats_get(evdev,
1074                                         RTE_EVENT_DEV_XSTATS_PORT,
1075                                         0, ids, values, num_stats);
1076         for (i = 0; (signed int)i < ret; i++) {
1077                 if (port_expected_zero[i] != values[i]) {
1078                         printf(
1079                                 "%d, Error, xstat %d (id %d) %s : %"PRIu64
1080                                 ", expect %"PRIu64"\n",
1081                                 __LINE__, i, ids[i], xstats_names[i].name,
1082                                 values[i], port_expected_zero[i]);
1083                         goto fail;
1084                 }
1085         }
1086
1087         /* QUEUE STATS TESTS */
1088         num_stats = rte_event_dev_xstats_names_get(evdev,
1089                                                 RTE_EVENT_DEV_XSTATS_QUEUE, 0,
1090                                                 xstats_names, ids, XSTATS_MAX);
1091         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE,
1092                                         0, ids, values, num_stats);
1093         if (ret < 0) {
1094                 printf("xstats get returned %d\n", ret);
1095                 goto fail;
1096         }
1097         if ((unsigned int)ret > XSTATS_MAX)
1098                 printf("%s %d: more xstats available than space\n",
1099                                 __func__, __LINE__);
1100
1101         static const uint64_t queue_expected[] = {
1102                 3 /* rx */,
1103                 3 /* tx */,
1104                 0 /* drop */,
1105                 3 /* inflights */,
1106                 512 /* iq size */,
1107                 0, 0, 0, 0, /* iq 0, 1, 2, 3 used */
1108                 /* QID-to-Port: pinned_flows, packets */
1109                 0, 0,
1110                 0, 0,
1111                 1, 3,
1112                 0, 0,
1113         };
1114         for (i = 0; (signed int)i < ret; i++) {
1115                 if (queue_expected[i] != values[i]) {
1116                         printf(
1117                                 "%d, Error, xstat %d (id %d) %s : %"PRIu64
1118                                 ", expect %"PRIu64"\n",
1119                                 __LINE__, i, ids[i], xstats_names[i].name,
1120                                 values[i], queue_expected[i]);
1121                         goto fail;
1122                 }
1123         }
1124
1125         /* Reset the queue stats here */
1126         ret = rte_event_dev_xstats_reset(evdev,
1127                                         RTE_EVENT_DEV_XSTATS_QUEUE, 0,
1128                                         NULL,
1129                                         0);
1130
1131         /* Verify that the resetable stats are reset, and others are not */
1132         static const uint64_t queue_expected_zero[] = {
1133                 0 /* rx */,
1134                 0 /* tx */,
1135                 0 /* drop */,
1136                 3 /* inflight */,
1137                 512 /* iq size */,
1138                 0, 0, 0, 0, /* 4 iq used */
1139                 /* QID-to-Port: pinned_flows, packets */
1140                 0, 0,
1141                 0, 0,
1142                 1, 0,
1143                 0, 0,
1144         };
1145
1146         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE, 0,
1147                                         ids, values, num_stats);
1148         int fails = 0;
1149         for (i = 0; (signed int)i < ret; i++) {
1150                 if (queue_expected_zero[i] != values[i]) {
1151                         printf(
1152                                 "%d, Error, xstat %d (id %d) %s : %"PRIu64
1153                                 ", expect %"PRIu64"\n",
1154                                 __LINE__, i, ids[i], xstats_names[i].name,
1155                                 values[i], queue_expected_zero[i]);
1156                         fails++;
1157                 }
1158         }
1159         if (fails) {
1160                 printf("%d : %d of values were not as expected above\n",
1161                                 __LINE__, fails);
1162                 goto fail;
1163         }
1164
1165         cleanup(t);
1166         return 0;
1167
1168 fail:
1169         rte_event_dev_dump(0, stdout);
1170         cleanup(t);
1171         return -1;
1172 }
1173
1174
1175 static int
1176 xstats_id_abuse_tests(struct test *t)
1177 {
1178         int err;
1179         const uint32_t XSTATS_MAX = 1024;
1180         const uint32_t link_port = 2;
1181
1182         uint32_t ids[XSTATS_MAX];
1183         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
1184
1185         /* Create instance with 4 ports */
1186         if (init(t, 1, 4) < 0 ||
1187                         create_ports(t, 4) < 0 ||
1188                         create_atomic_qids(t, 1) < 0) {
1189                 printf("%d: Error initializing device\n", __LINE__);
1190                 goto fail;
1191         }
1192
1193         err = rte_event_port_link(evdev, t->port[link_port], NULL, NULL, 0);
1194         if (err != 1) {
1195                 printf("%d: error mapping lb qid\n", __LINE__);
1196                 goto fail;
1197         }
1198
1199         if (rte_event_dev_start(evdev) < 0) {
1200                 printf("%d: Error with start call\n", __LINE__);
1201                 goto fail;
1202         }
1203
1204         /* no test for device, as it ignores the port/q number */
1205         int num_stats = rte_event_dev_xstats_names_get(evdev,
1206                                         RTE_EVENT_DEV_XSTATS_PORT,
1207                                         UINT8_MAX-1, xstats_names, ids,
1208                                         XSTATS_MAX);
1209         if (num_stats != 0) {
1210                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1211                                 0, num_stats);
1212                 goto fail;
1213         }
1214
1215         num_stats = rte_event_dev_xstats_names_get(evdev,
1216                                         RTE_EVENT_DEV_XSTATS_QUEUE,
1217                                         UINT8_MAX-1, xstats_names, ids,
1218                                         XSTATS_MAX);
1219         if (num_stats != 0) {
1220                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1221                                 0, num_stats);
1222                 goto fail;
1223         }
1224
1225         cleanup(t);
1226         return 0;
1227 fail:
1228         cleanup(t);
1229         return -1;
1230 }
1231
1232 static int
1233 port_reconfig_credits(struct test *t)
1234 {
1235         if (init(t, 1, 1) < 0) {
1236                 printf("%d: Error initializing device\n", __LINE__);
1237                 return -1;
1238         }
1239
1240         uint32_t i;
1241         const uint32_t NUM_ITERS = 32;
1242         for (i = 0; i < NUM_ITERS; i++) {
1243                 const struct rte_event_queue_conf conf = {
1244                         .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1245                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1246                         .nb_atomic_flows = 1024,
1247                         .nb_atomic_order_sequences = 1024,
1248                 };
1249                 if (rte_event_queue_setup(evdev, 0, &conf) < 0) {
1250                         printf("%d: error creating qid\n", __LINE__);
1251                         return -1;
1252                 }
1253                 t->qid[0] = 0;
1254
1255                 static const struct rte_event_port_conf port_conf = {
1256                                 .new_event_threshold = 128,
1257                                 .dequeue_depth = 32,
1258                                 .enqueue_depth = 64,
1259                 };
1260                 if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
1261                         printf("%d Error setting up port\n", __LINE__);
1262                         return -1;
1263                 }
1264
1265                 int links = rte_event_port_link(evdev, 0, NULL, NULL, 0);
1266                 if (links != 1) {
1267                         printf("%d: error mapping lb qid\n", __LINE__);
1268                         goto fail;
1269                 }
1270
1271                 if (rte_event_dev_start(evdev) < 0) {
1272                         printf("%d: Error with start call\n", __LINE__);
1273                         goto fail;
1274                 }
1275
1276                 const uint32_t NPKTS = 1;
1277                 uint32_t j;
1278                 for (j = 0; j < NPKTS; j++) {
1279                         struct rte_event ev;
1280                         struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1281                         if (!arp) {
1282                                 printf("%d: gen of pkt failed\n", __LINE__);
1283                                 goto fail;
1284                         }
1285                         ev.queue_id = t->qid[0];
1286                         ev.op = RTE_EVENT_OP_NEW;
1287                         ev.mbuf = arp;
1288                         int err = rte_event_enqueue_burst(evdev, 0, &ev, 1);
1289                         if (err != 1) {
1290                                 printf("%d: Failed to enqueue\n", __LINE__);
1291                                 rte_event_dev_dump(0, stdout);
1292                                 goto fail;
1293                         }
1294                 }
1295
1296                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
1297
1298                 struct rte_event ev[NPKTS];
1299                 int deq = rte_event_dequeue_burst(evdev, t->port[0], ev,
1300                                                         NPKTS, 0);
1301                 if (deq != 1)
1302                         printf("%d error; no packet dequeued\n", __LINE__);
1303
1304                 /* let cleanup below stop the device on last iter */
1305                 if (i != NUM_ITERS-1)
1306                         rte_event_dev_stop(evdev);
1307         }
1308
1309         cleanup(t);
1310         return 0;
1311 fail:
1312         cleanup(t);
1313         return -1;
1314 }
1315
1316 static int
1317 port_single_lb_reconfig(struct test *t)
1318 {
1319         if (init(t, 2, 2) < 0) {
1320                 printf("%d: Error initializing device\n", __LINE__);
1321                 goto fail;
1322         }
1323
1324         static const struct rte_event_queue_conf conf_lb_atomic = {
1325                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1326                 .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1327                 .nb_atomic_flows = 1024,
1328                 .nb_atomic_order_sequences = 1024,
1329         };
1330         if (rte_event_queue_setup(evdev, 0, &conf_lb_atomic) < 0) {
1331                 printf("%d: error creating qid\n", __LINE__);
1332                 goto fail;
1333         }
1334
1335         static const struct rte_event_queue_conf conf_single_link = {
1336                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1337                 .event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK,
1338         };
1339         if (rte_event_queue_setup(evdev, 1, &conf_single_link) < 0) {
1340                 printf("%d: error creating qid\n", __LINE__);
1341                 goto fail;
1342         }
1343
1344         struct rte_event_port_conf port_conf = {
1345                 .new_event_threshold = 128,
1346                 .dequeue_depth = 32,
1347                 .enqueue_depth = 64,
1348         };
1349         if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
1350                 printf("%d Error setting up port\n", __LINE__);
1351                 goto fail;
1352         }
1353         if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
1354                 printf("%d Error setting up port\n", __LINE__);
1355                 goto fail;
1356         }
1357
1358         /* link port to lb queue */
1359         uint8_t queue_id = 0;
1360         if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
1361                 printf("%d: error creating link for qid\n", __LINE__);
1362                 goto fail;
1363         }
1364
1365         int ret = rte_event_port_unlink(evdev, 0, &queue_id, 1);
1366         if (ret != 1) {
1367                 printf("%d: Error unlinking lb port\n", __LINE__);
1368                 goto fail;
1369         }
1370
1371         queue_id = 1;
1372         if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
1373                 printf("%d: error creating link for qid\n", __LINE__);
1374                 goto fail;
1375         }
1376
1377         queue_id = 0;
1378         int err = rte_event_port_link(evdev, 1, &queue_id, NULL, 1);
1379         if (err != 1) {
1380                 printf("%d: error mapping lb qid\n", __LINE__);
1381                 goto fail;
1382         }
1383
1384         if (rte_event_dev_start(evdev) < 0) {
1385                 printf("%d: Error with start call\n", __LINE__);
1386                 goto fail;
1387         }
1388
1389         cleanup(t);
1390         return 0;
1391 fail:
1392         cleanup(t);
1393         return -1;
1394 }
1395
1396 static int
1397 xstats_brute_force(struct test *t)
1398 {
1399         uint32_t i;
1400         const uint32_t XSTATS_MAX = 1024;
1401         uint32_t ids[XSTATS_MAX];
1402         uint64_t values[XSTATS_MAX];
1403         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
1404
1405
1406         /* Create instance with 4 ports */
1407         if (init(t, 1, 4) < 0 ||
1408                         create_ports(t, 4) < 0 ||
1409                         create_atomic_qids(t, 1) < 0) {
1410                 printf("%d: Error initializing device\n", __LINE__);
1411                 return -1;
1412         }
1413
1414         int err = rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1415         if (err != 1) {
1416                 printf("%d: error mapping lb qid\n", __LINE__);
1417                 goto fail;
1418         }
1419
1420         if (rte_event_dev_start(evdev) < 0) {
1421                 printf("%d: Error with start call\n", __LINE__);
1422                 goto fail;
1423         }
1424
1425         for (i = 0; i < XSTATS_MAX; i++)
1426                 ids[i] = i;
1427
1428         for (i = 0; i < 3; i++) {
1429                 uint32_t mode = RTE_EVENT_DEV_XSTATS_DEVICE + i;
1430                 uint32_t j;
1431                 for (j = 0; j < UINT8_MAX; j++) {
1432                         rte_event_dev_xstats_names_get(evdev, mode,
1433                                 j, xstats_names, ids, XSTATS_MAX);
1434
1435                         rte_event_dev_xstats_get(evdev, mode, j, ids,
1436                                                  values, XSTATS_MAX);
1437                 }
1438         }
1439
1440         cleanup(t);
1441         return 0;
1442 fail:
1443         cleanup(t);
1444         return -1;
1445 }
1446
1447 static int
1448 xstats_id_reset_tests(struct test *t)
1449 {
1450         const int wrk_enq = 2;
1451         int err;
1452
1453         /* Create instance with 4 ports */
1454         if (init(t, 1, 4) < 0 ||
1455                         create_ports(t, 4) < 0 ||
1456                         create_atomic_qids(t, 1) < 0) {
1457                 printf("%d: Error initializing device\n", __LINE__);
1458                 return -1;
1459         }
1460
1461         /* CQ mapping to QID */
1462         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
1463         if (err != 1) {
1464                 printf("%d: error mapping lb qid\n", __LINE__);
1465                 goto fail;
1466         }
1467
1468         if (rte_event_dev_start(evdev) < 0) {
1469                 printf("%d: Error with start call\n", __LINE__);
1470                 goto fail;
1471         }
1472
1473 #define XSTATS_MAX 1024
1474         int ret;
1475         uint32_t i;
1476         uint32_t ids[XSTATS_MAX];
1477         uint64_t values[XSTATS_MAX];
1478         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
1479
1480         for (i = 0; i < XSTATS_MAX; i++)
1481                 ids[i] = i;
1482
1483 #define NUM_DEV_STATS 6
1484         /* Device names / values */
1485         int num_stats = rte_event_dev_xstats_names_get(evdev,
1486                                         RTE_EVENT_DEV_XSTATS_DEVICE,
1487                                         0, xstats_names, ids, XSTATS_MAX);
1488         if (num_stats != NUM_DEV_STATS) {
1489                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1490                                 NUM_DEV_STATS, num_stats);
1491                 goto fail;
1492         }
1493         ret = rte_event_dev_xstats_get(evdev,
1494                                         RTE_EVENT_DEV_XSTATS_DEVICE,
1495                                         0, ids, values, num_stats);
1496         if (ret != NUM_DEV_STATS) {
1497                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1498                                 NUM_DEV_STATS, ret);
1499                 goto fail;
1500         }
1501
1502 #define NPKTS 7
1503         for (i = 0; i < NPKTS; i++) {
1504                 struct rte_event ev;
1505                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1506                 if (!arp) {
1507                         printf("%d: gen of pkt failed\n", __LINE__);
1508                         goto fail;
1509                 }
1510                 ev.queue_id = t->qid[i];
1511                 ev.op = RTE_EVENT_OP_NEW;
1512                 ev.mbuf = arp;
1513                 arp->seqn = i;
1514
1515                 int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
1516                 if (err != 1) {
1517                         printf("%d: Failed to enqueue\n", __LINE__);
1518                         goto fail;
1519                 }
1520         }
1521
1522         rte_service_run_iter_on_app_lcore(t->service_id, 1);
1523
1524         static const char * const dev_names[] = {
1525                 "dev_rx", "dev_tx", "dev_drop", "dev_sched_calls",
1526                 "dev_sched_no_iq_enq", "dev_sched_no_cq_enq",
1527         };
1528         uint64_t dev_expected[] = {NPKTS, NPKTS, 0, 1, 0, 0};
1529         for (i = 0; (int)i < ret; i++) {
1530                 unsigned int id;
1531                 uint64_t val = rte_event_dev_xstats_by_name_get(evdev,
1532                                                                 dev_names[i],
1533                                                                 &id);
1534                 if (id != i) {
1535                         printf("%d: %s id incorrect, expected %d got %d\n",
1536                                         __LINE__, dev_names[i], i, id);
1537                         goto fail;
1538                 }
1539                 if (val != dev_expected[i]) {
1540                         printf("%d: %s value incorrect, expected %"
1541                                 PRIu64" got %d\n", __LINE__, dev_names[i],
1542                                 dev_expected[i], id);
1543                         goto fail;
1544                 }
1545                 /* reset to zero */
1546                 int reset_ret = rte_event_dev_xstats_reset(evdev,
1547                                                 RTE_EVENT_DEV_XSTATS_DEVICE, 0,
1548                                                 &id,
1549                                                 1);
1550                 if (reset_ret) {
1551                         printf("%d: failed to reset successfully\n", __LINE__);
1552                         goto fail;
1553                 }
1554                 dev_expected[i] = 0;
1555                 /* check value again */
1556                 val = rte_event_dev_xstats_by_name_get(evdev, dev_names[i], 0);
1557                 if (val != dev_expected[i]) {
1558                         printf("%d: %s value incorrect, expected %"PRIu64
1559                                 " got %"PRIu64"\n", __LINE__, dev_names[i],
1560                                 dev_expected[i], val);
1561                         goto fail;
1562                 }
1563         };
1564
1565 /* 48 is stat offset from start of the devices whole xstats.
1566  * This WILL break every time we add a statistic to a port
1567  * or the device, but there is no other way to test
1568  */
1569 #define PORT_OFF 48
1570 /* num stats for the tested port. CQ size adds more stats to a port */
1571 #define NUM_PORT_STATS 21
1572 /* the port to test. */
1573 #define PORT 2
1574         num_stats = rte_event_dev_xstats_names_get(evdev,
1575                                         RTE_EVENT_DEV_XSTATS_PORT, PORT,
1576                                         xstats_names, ids, XSTATS_MAX);
1577         if (num_stats != NUM_PORT_STATS) {
1578                 printf("%d: expected %d stats, got return %d\n",
1579                         __LINE__, NUM_PORT_STATS, num_stats);
1580                 goto fail;
1581         }
1582         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_PORT, PORT,
1583                                         ids, values, num_stats);
1584
1585         if (ret != NUM_PORT_STATS) {
1586                 printf("%d: expected %d stats, got return %d\n",
1587                                 __LINE__, NUM_PORT_STATS, ret);
1588                 goto fail;
1589         }
1590         static const char * const port_names[] = {
1591                 "port_2_rx",
1592                 "port_2_tx",
1593                 "port_2_drop",
1594                 "port_2_inflight",
1595                 "port_2_avg_pkt_cycles",
1596                 "port_2_credits",
1597                 "port_2_rx_ring_used",
1598                 "port_2_rx_ring_free",
1599                 "port_2_cq_ring_used",
1600                 "port_2_cq_ring_free",
1601                 "port_2_dequeue_calls",
1602                 "port_2_dequeues_returning_0",
1603                 "port_2_dequeues_returning_1-4",
1604                 "port_2_dequeues_returning_5-8",
1605                 "port_2_dequeues_returning_9-12",
1606                 "port_2_dequeues_returning_13-16",
1607                 "port_2_dequeues_returning_17-20",
1608                 "port_2_dequeues_returning_21-24",
1609                 "port_2_dequeues_returning_25-28",
1610                 "port_2_dequeues_returning_29-32",
1611                 "port_2_dequeues_returning_33-36",
1612         };
1613         uint64_t port_expected[] = {
1614                 0, /* rx */
1615                 NPKTS, /* tx */
1616                 0, /* drop */
1617                 NPKTS, /* inflight */
1618                 0, /* avg pkt cycles */
1619                 0, /* credits */
1620                 0, /* rx ring used */
1621                 4096, /* rx ring free */
1622                 NPKTS,  /* cq ring used */
1623                 25, /* cq ring free */
1624                 0, /* dequeue zero calls */
1625                 0, 0, 0, 0, 0, /* 10 dequeue buckets */
1626                 0, 0, 0, 0, 0,
1627         };
1628         uint64_t port_expected_zero[] = {
1629                 0, /* rx */
1630                 0, /* tx */
1631                 0, /* drop */
1632                 NPKTS, /* inflight */
1633                 0, /* avg pkt cycles */
1634                 0, /* credits */
1635                 0, /* rx ring used */
1636                 4096, /* rx ring free */
1637                 NPKTS,  /* cq ring used */
1638                 25, /* cq ring free */
1639                 0, /* dequeue zero calls */
1640                 0, 0, 0, 0, 0, /* 10 dequeue buckets */
1641                 0, 0, 0, 0, 0,
1642         };
1643         if (RTE_DIM(port_expected) != NUM_PORT_STATS ||
1644                         RTE_DIM(port_names) != NUM_PORT_STATS) {
1645                 printf("%d: port array of wrong size\n", __LINE__);
1646                 goto fail;
1647         }
1648
1649         int failed = 0;
1650         for (i = 0; (int)i < ret; i++) {
1651                 unsigned int id;
1652                 uint64_t val = rte_event_dev_xstats_by_name_get(evdev,
1653                                                                 port_names[i],
1654                                                                 &id);
1655                 if (id != i + PORT_OFF) {
1656                         printf("%d: %s id incorrect, expected %d got %d\n",
1657                                         __LINE__, port_names[i], i+PORT_OFF,
1658                                         id);
1659                         failed = 1;
1660                 }
1661                 if (val != port_expected[i]) {
1662                         printf("%d: %s value incorrect, expected %"PRIu64
1663                                 " got %d\n", __LINE__, port_names[i],
1664                                 port_expected[i], id);
1665                         failed = 1;
1666                 }
1667                 /* reset to zero */
1668                 int reset_ret = rte_event_dev_xstats_reset(evdev,
1669                                                 RTE_EVENT_DEV_XSTATS_PORT, PORT,
1670                                                 &id,
1671                                                 1);
1672                 if (reset_ret) {
1673                         printf("%d: failed to reset successfully\n", __LINE__);
1674                         failed = 1;
1675                 }
1676                 /* check value again */
1677                 val = rte_event_dev_xstats_by_name_get(evdev, port_names[i], 0);
1678                 if (val != port_expected_zero[i]) {
1679                         printf("%d: %s value incorrect, expected %"PRIu64
1680                                 " got %"PRIu64"\n", __LINE__, port_names[i],
1681                                 port_expected_zero[i], val);
1682                         failed = 1;
1683                 }
1684         };
1685         if (failed)
1686                 goto fail;
1687
1688 /* num queue stats */
1689 #define NUM_Q_STATS 17
1690 /* queue offset from start of the devices whole xstats.
1691  * This will break every time we add a statistic to a device/port/queue
1692  */
1693 #define QUEUE_OFF 90
1694         const uint32_t queue = 0;
1695         num_stats = rte_event_dev_xstats_names_get(evdev,
1696                                         RTE_EVENT_DEV_XSTATS_QUEUE, queue,
1697                                         xstats_names, ids, XSTATS_MAX);
1698         if (num_stats != NUM_Q_STATS) {
1699                 printf("%d: expected %d stats, got return %d\n",
1700                         __LINE__, NUM_Q_STATS, num_stats);
1701                 goto fail;
1702         }
1703         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE,
1704                                         queue, ids, values, num_stats);
1705         if (ret != NUM_Q_STATS) {
1706                 printf("%d: expected 21 stats, got return %d\n", __LINE__, ret);
1707                 goto fail;
1708         }
1709         static const char * const queue_names[] = {
1710                 "qid_0_rx",
1711                 "qid_0_tx",
1712                 "qid_0_drop",
1713                 "qid_0_inflight",
1714                 "qid_0_iq_size",
1715                 "qid_0_iq_0_used",
1716                 "qid_0_iq_1_used",
1717                 "qid_0_iq_2_used",
1718                 "qid_0_iq_3_used",
1719                 "qid_0_port_0_pinned_flows",
1720                 "qid_0_port_0_packets",
1721                 "qid_0_port_1_pinned_flows",
1722                 "qid_0_port_1_packets",
1723                 "qid_0_port_2_pinned_flows",
1724                 "qid_0_port_2_packets",
1725                 "qid_0_port_3_pinned_flows",
1726                 "qid_0_port_3_packets",
1727         };
1728         uint64_t queue_expected[] = {
1729                 7, /* rx */
1730                 7, /* tx */
1731                 0, /* drop */
1732                 7, /* inflight */
1733                 512, /* iq size */
1734                 0, /* iq 0 used */
1735                 0, /* iq 1 used */
1736                 0, /* iq 2 used */
1737                 0, /* iq 3 used */
1738                 /* QID-to-Port: pinned_flows, packets */
1739                 0, 0,
1740                 0, 0,
1741                 1, 7,
1742                 0, 0,
1743         };
1744         uint64_t queue_expected_zero[] = {
1745                 0, /* rx */
1746                 0, /* tx */
1747                 0, /* drop */
1748                 7, /* inflight */
1749                 512, /* iq size */
1750                 0, /* iq 0 used */
1751                 0, /* iq 1 used */
1752                 0, /* iq 2 used */
1753                 0, /* iq 3 used */
1754                 /* QID-to-Port: pinned_flows, packets */
1755                 0, 0,
1756                 0, 0,
1757                 1, 0,
1758                 0, 0,
1759         };
1760         if (RTE_DIM(queue_expected) != NUM_Q_STATS ||
1761                         RTE_DIM(queue_expected_zero) != NUM_Q_STATS ||
1762                         RTE_DIM(queue_names) != NUM_Q_STATS) {
1763                 printf("%d : queue array of wrong size\n", __LINE__);
1764                 goto fail;
1765         }
1766
1767         failed = 0;
1768         for (i = 0; (int)i < ret; i++) {
1769                 unsigned int id;
1770                 uint64_t val = rte_event_dev_xstats_by_name_get(evdev,
1771                                                                 queue_names[i],
1772                                                                 &id);
1773                 if (id != i + QUEUE_OFF) {
1774                         printf("%d: %s id incorrect, expected %d got %d\n",
1775                                         __LINE__, queue_names[i], i+QUEUE_OFF,
1776                                         id);
1777                         failed = 1;
1778                 }
1779                 if (val != queue_expected[i]) {
1780                         printf("%d: %d: %s value , expected %"PRIu64
1781                                 " got %"PRIu64"\n", i, __LINE__,
1782                                 queue_names[i], queue_expected[i], val);
1783                         failed = 1;
1784                 }
1785                 /* reset to zero */
1786                 int reset_ret = rte_event_dev_xstats_reset(evdev,
1787                                                 RTE_EVENT_DEV_XSTATS_QUEUE,
1788                                                 queue, &id, 1);
1789                 if (reset_ret) {
1790                         printf("%d: failed to reset successfully\n", __LINE__);
1791                         failed = 1;
1792                 }
1793                 /* check value again */
1794                 val = rte_event_dev_xstats_by_name_get(evdev, queue_names[i],
1795                                                         0);
1796                 if (val != queue_expected_zero[i]) {
1797                         printf("%d: %s value incorrect, expected %"PRIu64
1798                                 " got %"PRIu64"\n", __LINE__, queue_names[i],
1799                                 queue_expected_zero[i], val);
1800                         failed = 1;
1801                 }
1802         };
1803
1804         if (failed)
1805                 goto fail;
1806
1807         cleanup(t);
1808         return 0;
1809 fail:
1810         cleanup(t);
1811         return -1;
1812 }
1813
1814 static int
1815 ordered_reconfigure(struct test *t)
1816 {
1817         if (init(t, 1, 1) < 0 ||
1818                         create_ports(t, 1) < 0) {
1819                 printf("%d: Error initializing device\n", __LINE__);
1820                 return -1;
1821         }
1822
1823         const struct rte_event_queue_conf conf = {
1824                         .schedule_type = RTE_SCHED_TYPE_ORDERED,
1825                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1826                         .nb_atomic_flows = 1024,
1827                         .nb_atomic_order_sequences = 1024,
1828         };
1829
1830         if (rte_event_queue_setup(evdev, 0, &conf) < 0) {
1831                 printf("%d: error creating qid\n", __LINE__);
1832                 goto failed;
1833         }
1834
1835         if (rte_event_queue_setup(evdev, 0, &conf) < 0) {
1836                 printf("%d: error creating qid, for 2nd time\n", __LINE__);
1837                 goto failed;
1838         }
1839
1840         rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1841         if (rte_event_dev_start(evdev) < 0) {
1842                 printf("%d: Error with start call\n", __LINE__);
1843                 return -1;
1844         }
1845
1846         cleanup(t);
1847         return 0;
1848 failed:
1849         cleanup(t);
1850         return -1;
1851 }
1852
1853 static int
1854 qid_priorities(struct test *t)
1855 {
1856         /* Test works by having a CQ with enough empty space for all packets,
1857          * and enqueueing 3 packets to 3 QIDs. They must return based on the
1858          * priority of the QID, not the ingress order, to pass the test
1859          */
1860         unsigned int i;
1861         /* Create instance with 1 ports, and 3 qids */
1862         if (init(t, 3, 1) < 0 ||
1863                         create_ports(t, 1) < 0) {
1864                 printf("%d: Error initializing device\n", __LINE__);
1865                 return -1;
1866         }
1867
1868         for (i = 0; i < 3; i++) {
1869                 /* Create QID */
1870                 const struct rte_event_queue_conf conf = {
1871                         .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1872                         /* increase priority (0 == highest), as we go */
1873                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL - i,
1874                         .nb_atomic_flows = 1024,
1875                         .nb_atomic_order_sequences = 1024,
1876                 };
1877
1878                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
1879                         printf("%d: error creating qid %d\n", __LINE__, i);
1880                         return -1;
1881                 }
1882                 t->qid[i] = i;
1883         }
1884         t->nb_qids = i;
1885         /* map all QIDs to port */
1886         rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1887
1888         if (rte_event_dev_start(evdev) < 0) {
1889                 printf("%d: Error with start call\n", __LINE__);
1890                 return -1;
1891         }
1892
1893         /* enqueue 3 packets, setting seqn and QID to check priority */
1894         for (i = 0; i < 3; i++) {
1895                 struct rte_event ev;
1896                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1897                 if (!arp) {
1898                         printf("%d: gen of pkt failed\n", __LINE__);
1899                         return -1;
1900                 }
1901                 ev.queue_id = t->qid[i];
1902                 ev.op = RTE_EVENT_OP_NEW;
1903                 ev.mbuf = arp;
1904                 arp->seqn = i;
1905
1906                 int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
1907                 if (err != 1) {
1908                         printf("%d: Failed to enqueue\n", __LINE__);
1909                         return -1;
1910                 }
1911         }
1912
1913         rte_service_run_iter_on_app_lcore(t->service_id, 1);
1914
1915         /* dequeue packets, verify priority was upheld */
1916         struct rte_event ev[32];
1917         uint32_t deq_pkts =
1918                 rte_event_dequeue_burst(evdev, t->port[0], ev, 32, 0);
1919         if (deq_pkts != 3) {
1920                 printf("%d: failed to deq packets\n", __LINE__);
1921                 rte_event_dev_dump(evdev, stdout);
1922                 return -1;
1923         }
1924         for (i = 0; i < 3; i++) {
1925                 if (ev[i].mbuf->seqn != 2-i) {
1926                         printf(
1927                                 "%d: qid priority test: seqn %d incorrectly prioritized\n",
1928                                         __LINE__, i);
1929                 }
1930         }
1931
1932         cleanup(t);
1933         return 0;
1934 }
1935
1936 static int
1937 load_balancing(struct test *t)
1938 {
1939         const int rx_enq = 0;
1940         int err;
1941         uint32_t i;
1942
1943         if (init(t, 1, 4) < 0 ||
1944                         create_ports(t, 4) < 0 ||
1945                         create_atomic_qids(t, 1) < 0) {
1946                 printf("%d: Error initializing device\n", __LINE__);
1947                 return -1;
1948         }
1949
1950         for (i = 0; i < 3; i++) {
1951                 /* map port 1 - 3 inclusive */
1952                 if (rte_event_port_link(evdev, t->port[i+1], &t->qid[0],
1953                                 NULL, 1) != 1) {
1954                         printf("%d: error mapping qid to port %d\n",
1955                                         __LINE__, i);
1956                         return -1;
1957                 }
1958         }
1959
1960         if (rte_event_dev_start(evdev) < 0) {
1961                 printf("%d: Error with start call\n", __LINE__);
1962                 return -1;
1963         }
1964
1965         /************** FORWARD ****************/
1966         /*
1967          * Create a set of flows that test the load-balancing operation of the
1968          * implementation. Fill CQ 0 and 1 with flows 0 and 1, and test
1969          * with a new flow, which should be sent to the 3rd mapped CQ
1970          */
1971         static uint32_t flows[] = {0, 1, 1, 0, 0, 2, 2, 0, 2};
1972
1973         for (i = 0; i < RTE_DIM(flows); i++) {
1974                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1975                 if (!arp) {
1976                         printf("%d: gen of pkt failed\n", __LINE__);
1977                         return -1;
1978                 }
1979
1980                 struct rte_event ev = {
1981                                 .op = RTE_EVENT_OP_NEW,
1982                                 .queue_id = t->qid[0],
1983                                 .flow_id = flows[i],
1984                                 .mbuf = arp,
1985                 };
1986                 /* generate pkt and enqueue */
1987                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
1988                 if (err < 0) {
1989                         printf("%d: Failed to enqueue\n", __LINE__);
1990                         return -1;
1991                 }
1992         }
1993
1994         rte_service_run_iter_on_app_lcore(t->service_id, 1);
1995
1996         struct test_event_dev_stats stats;
1997         err = test_event_dev_stats_get(evdev, &stats);
1998         if (err) {
1999                 printf("%d: failed to get stats\n", __LINE__);
2000                 return -1;
2001         }
2002
2003         if (stats.port_inflight[1] != 4) {
2004                 printf("%d:%s: port 1 inflight not correct\n", __LINE__,
2005                                 __func__);
2006                 return -1;
2007         }
2008         if (stats.port_inflight[2] != 2) {
2009                 printf("%d:%s: port 2 inflight not correct\n", __LINE__,
2010                                 __func__);
2011                 return -1;
2012         }
2013         if (stats.port_inflight[3] != 3) {
2014                 printf("%d:%s: port 3 inflight not correct\n", __LINE__,
2015                                 __func__);
2016                 return -1;
2017         }
2018
2019         cleanup(t);
2020         return 0;
2021 }
2022
2023 static int
2024 load_balancing_history(struct test *t)
2025 {
2026         struct test_event_dev_stats stats = {0};
2027         const int rx_enq = 0;
2028         int err;
2029         uint32_t i;
2030
2031         /* Create instance with 1 atomic QID going to 3 ports + 1 prod port */
2032         if (init(t, 1, 4) < 0 ||
2033                         create_ports(t, 4) < 0 ||
2034                         create_atomic_qids(t, 1) < 0)
2035                 return -1;
2036
2037         /* CQ mapping to QID */
2038         if (rte_event_port_link(evdev, t->port[1], &t->qid[0], NULL, 1) != 1) {
2039                 printf("%d: error mapping port 1 qid\n", __LINE__);
2040                 return -1;
2041         }
2042         if (rte_event_port_link(evdev, t->port[2], &t->qid[0], NULL, 1) != 1) {
2043                 printf("%d: error mapping port 2 qid\n", __LINE__);
2044                 return -1;
2045         }
2046         if (rte_event_port_link(evdev, t->port[3], &t->qid[0], NULL, 1) != 1) {
2047                 printf("%d: error mapping port 3 qid\n", __LINE__);
2048                 return -1;
2049         }
2050         if (rte_event_dev_start(evdev) < 0) {
2051                 printf("%d: Error with start call\n", __LINE__);
2052                 return -1;
2053         }
2054
2055         /*
2056          * Create a set of flows that test the load-balancing operation of the
2057          * implementation. Fill CQ 0, 1 and 2 with flows 0, 1 and 2, drop
2058          * the packet from CQ 0, send in a new set of flows. Ensure that:
2059          *  1. The new flow 3 gets into the empty CQ0
2060          *  2. packets for existing flow gets added into CQ1
2061          *  3. Next flow 0 pkt is now onto CQ2, since CQ0 and CQ1 now contain
2062          *     more outstanding pkts
2063          *
2064          *  This test makes sure that when a flow ends (i.e. all packets
2065          *  have been completed for that flow), that the flow can be moved
2066          *  to a different CQ when new packets come in for that flow.
2067          */
2068         static uint32_t flows1[] = {0, 1, 1, 2};
2069
2070         for (i = 0; i < RTE_DIM(flows1); i++) {
2071                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2072                 struct rte_event ev = {
2073                                 .flow_id = flows1[i],
2074                                 .op = RTE_EVENT_OP_NEW,
2075                                 .queue_id = t->qid[0],
2076                                 .event_type = RTE_EVENT_TYPE_CPU,
2077                                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
2078                                 .mbuf = arp
2079                 };
2080
2081                 if (!arp) {
2082                         printf("%d: gen of pkt failed\n", __LINE__);
2083                         return -1;
2084                 }
2085                 arp->hash.rss = flows1[i];
2086                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2087                 if (err < 0) {
2088                         printf("%d: Failed to enqueue\n", __LINE__);
2089                         return -1;
2090                 }
2091         }
2092
2093         /* call the scheduler */
2094         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2095
2096         /* Dequeue the flow 0 packet from port 1, so that we can then drop */
2097         struct rte_event ev;
2098         if (!rte_event_dequeue_burst(evdev, t->port[1], &ev, 1, 0)) {
2099                 printf("%d: failed to dequeue\n", __LINE__);
2100                 return -1;
2101         }
2102         if (ev.mbuf->hash.rss != flows1[0]) {
2103                 printf("%d: unexpected flow received\n", __LINE__);
2104                 return -1;
2105         }
2106
2107         /* drop the flow 0 packet from port 1 */
2108         rte_event_enqueue_burst(evdev, t->port[1], &release_ev, 1);
2109
2110         /* call the scheduler */
2111         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2112
2113         /*
2114          * Set up the next set of flows, first a new flow to fill up
2115          * CQ 0, so that the next flow 0 packet should go to CQ2
2116          */
2117         static uint32_t flows2[] = { 3, 3, 3, 1, 1, 0 };
2118
2119         for (i = 0; i < RTE_DIM(flows2); i++) {
2120                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2121                 struct rte_event ev = {
2122                                 .flow_id = flows2[i],
2123                                 .op = RTE_EVENT_OP_NEW,
2124                                 .queue_id = t->qid[0],
2125                                 .event_type = RTE_EVENT_TYPE_CPU,
2126                                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
2127                                 .mbuf = arp
2128                 };
2129
2130                 if (!arp) {
2131                         printf("%d: gen of pkt failed\n", __LINE__);
2132                         return -1;
2133                 }
2134                 arp->hash.rss = flows2[i];
2135
2136                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2137                 if (err < 0) {
2138                         printf("%d: Failed to enqueue\n", __LINE__);
2139                         return -1;
2140                 }
2141         }
2142
2143         /* schedule */
2144         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2145
2146         err = test_event_dev_stats_get(evdev, &stats);
2147         if (err) {
2148                 printf("%d:failed to get stats\n", __LINE__);
2149                 return -1;
2150         }
2151
2152         /*
2153          * Now check the resulting inflights on each port.
2154          */
2155         if (stats.port_inflight[1] != 3) {
2156                 printf("%d:%s: port 1 inflight not correct\n", __LINE__,
2157                                 __func__);
2158                 printf("Inflights, ports 1, 2, 3: %u, %u, %u\n",
2159                                 (unsigned int)stats.port_inflight[1],
2160                                 (unsigned int)stats.port_inflight[2],
2161                                 (unsigned int)stats.port_inflight[3]);
2162                 return -1;
2163         }
2164         if (stats.port_inflight[2] != 4) {
2165                 printf("%d:%s: port 2 inflight not correct\n", __LINE__,
2166                                 __func__);
2167                 printf("Inflights, ports 1, 2, 3: %u, %u, %u\n",
2168                                 (unsigned int)stats.port_inflight[1],
2169                                 (unsigned int)stats.port_inflight[2],
2170                                 (unsigned int)stats.port_inflight[3]);
2171                 return -1;
2172         }
2173         if (stats.port_inflight[3] != 2) {
2174                 printf("%d:%s: port 3 inflight not correct\n", __LINE__,
2175                                 __func__);
2176                 printf("Inflights, ports 1, 2, 3: %u, %u, %u\n",
2177                                 (unsigned int)stats.port_inflight[1],
2178                                 (unsigned int)stats.port_inflight[2],
2179                                 (unsigned int)stats.port_inflight[3]);
2180                 return -1;
2181         }
2182
2183         for (i = 1; i <= 3; i++) {
2184                 struct rte_event ev;
2185                 while (rte_event_dequeue_burst(evdev, i, &ev, 1, 0))
2186                         rte_event_enqueue_burst(evdev, i, &release_ev, 1);
2187         }
2188         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2189
2190         cleanup(t);
2191         return 0;
2192 }
2193
2194 static int
2195 invalid_qid(struct test *t)
2196 {
2197         struct test_event_dev_stats stats;
2198         const int rx_enq = 0;
2199         int err;
2200         uint32_t i;
2201
2202         if (init(t, 1, 4) < 0 ||
2203                         create_ports(t, 4) < 0 ||
2204                         create_atomic_qids(t, 1) < 0) {
2205                 printf("%d: Error initializing device\n", __LINE__);
2206                 return -1;
2207         }
2208
2209         /* CQ mapping to QID */
2210         for (i = 0; i < 4; i++) {
2211                 err = rte_event_port_link(evdev, t->port[i], &t->qid[0],
2212                                 NULL, 1);
2213                 if (err != 1) {
2214                         printf("%d: error mapping port 1 qid\n", __LINE__);
2215                         return -1;
2216                 }
2217         }
2218
2219         if (rte_event_dev_start(evdev) < 0) {
2220                 printf("%d: Error with start call\n", __LINE__);
2221                 return -1;
2222         }
2223
2224         /*
2225          * Send in a packet with an invalid qid to the scheduler.
2226          * We should see the packed enqueued OK, but the inflights for
2227          * that packet should not be incremented, and the rx_dropped
2228          * should be incremented.
2229          */
2230         static uint32_t flows1[] = {20};
2231
2232         for (i = 0; i < RTE_DIM(flows1); i++) {
2233                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2234                 if (!arp) {
2235                         printf("%d: gen of pkt failed\n", __LINE__);
2236                         return -1;
2237                 }
2238
2239                 struct rte_event ev = {
2240                                 .op = RTE_EVENT_OP_NEW,
2241                                 .queue_id = t->qid[0] + flows1[i],
2242                                 .flow_id = i,
2243                                 .mbuf = arp,
2244                 };
2245                 /* generate pkt and enqueue */
2246                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2247                 if (err < 0) {
2248                         printf("%d: Failed to enqueue\n", __LINE__);
2249                         return -1;
2250                 }
2251         }
2252
2253         /* call the scheduler */
2254         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2255
2256         err = test_event_dev_stats_get(evdev, &stats);
2257         if (err) {
2258                 printf("%d: failed to get stats\n", __LINE__);
2259                 return -1;
2260         }
2261
2262         /*
2263          * Now check the resulting inflights on the port, and the rx_dropped.
2264          */
2265         if (stats.port_inflight[0] != 0) {
2266                 printf("%d:%s: port 1 inflight count not correct\n", __LINE__,
2267                                 __func__);
2268                 rte_event_dev_dump(evdev, stdout);
2269                 return -1;
2270         }
2271         if (stats.port_rx_dropped[0] != 1) {
2272                 printf("%d:%s: port 1 drops\n", __LINE__, __func__);
2273                 rte_event_dev_dump(evdev, stdout);
2274                 return -1;
2275         }
2276         /* each packet drop should only be counted in one place - port or dev */
2277         if (stats.rx_dropped != 0) {
2278                 printf("%d:%s: port 1 dropped count not correct\n", __LINE__,
2279                                 __func__);
2280                 rte_event_dev_dump(evdev, stdout);
2281                 return -1;
2282         }
2283
2284         cleanup(t);
2285         return 0;
2286 }
2287
2288 static int
2289 single_packet(struct test *t)
2290 {
2291         const uint32_t MAGIC_SEQN = 7321;
2292         struct rte_event ev;
2293         struct test_event_dev_stats stats;
2294         const int rx_enq = 0;
2295         const int wrk_enq = 2;
2296         int err;
2297
2298         /* Create instance with 4 ports */
2299         if (init(t, 1, 4) < 0 ||
2300                         create_ports(t, 4) < 0 ||
2301                         create_atomic_qids(t, 1) < 0) {
2302                 printf("%d: Error initializing device\n", __LINE__);
2303                 return -1;
2304         }
2305
2306         /* CQ mapping to QID */
2307         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
2308         if (err != 1) {
2309                 printf("%d: error mapping lb qid\n", __LINE__);
2310                 cleanup(t);
2311                 return -1;
2312         }
2313
2314         if (rte_event_dev_start(evdev) < 0) {
2315                 printf("%d: Error with start call\n", __LINE__);
2316                 return -1;
2317         }
2318
2319         /************** Gen pkt and enqueue ****************/
2320         struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2321         if (!arp) {
2322                 printf("%d: gen of pkt failed\n", __LINE__);
2323                 return -1;
2324         }
2325
2326         ev.op = RTE_EVENT_OP_NEW;
2327         ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
2328         ev.mbuf = arp;
2329         ev.queue_id = 0;
2330         ev.flow_id = 3;
2331         arp->seqn = MAGIC_SEQN;
2332
2333         err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2334         if (err < 0) {
2335                 printf("%d: Failed to enqueue\n", __LINE__);
2336                 return -1;
2337         }
2338
2339         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2340
2341         err = test_event_dev_stats_get(evdev, &stats);
2342         if (err) {
2343                 printf("%d: failed to get stats\n", __LINE__);
2344                 return -1;
2345         }
2346
2347         if (stats.rx_pkts != 1 ||
2348                         stats.tx_pkts != 1 ||
2349                         stats.port_inflight[wrk_enq] != 1) {
2350                 printf("%d: Sched core didn't handle pkt as expected\n",
2351                                 __LINE__);
2352                 rte_event_dev_dump(evdev, stdout);
2353                 return -1;
2354         }
2355
2356         uint32_t deq_pkts;
2357
2358         deq_pkts = rte_event_dequeue_burst(evdev, t->port[wrk_enq], &ev, 1, 0);
2359         if (deq_pkts < 1) {
2360                 printf("%d: Failed to deq\n", __LINE__);
2361                 return -1;
2362         }
2363
2364         err = test_event_dev_stats_get(evdev, &stats);
2365         if (err) {
2366                 printf("%d: failed to get stats\n", __LINE__);
2367                 return -1;
2368         }
2369
2370         err = test_event_dev_stats_get(evdev, &stats);
2371         if (ev.mbuf->seqn != MAGIC_SEQN) {
2372                 printf("%d: magic sequence number not dequeued\n", __LINE__);
2373                 return -1;
2374         }
2375
2376         rte_pktmbuf_free(ev.mbuf);
2377         err = rte_event_enqueue_burst(evdev, t->port[wrk_enq], &release_ev, 1);
2378         if (err < 0) {
2379                 printf("%d: Failed to enqueue\n", __LINE__);
2380                 return -1;
2381         }
2382         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2383
2384         err = test_event_dev_stats_get(evdev, &stats);
2385         if (stats.port_inflight[wrk_enq] != 0) {
2386                 printf("%d: port inflight not correct\n", __LINE__);
2387                 return -1;
2388         }
2389
2390         cleanup(t);
2391         return 0;
2392 }
2393
2394 static int
2395 inflight_counts(struct test *t)
2396 {
2397         struct rte_event ev;
2398         struct test_event_dev_stats stats;
2399         const int rx_enq = 0;
2400         const int p1 = 1;
2401         const int p2 = 2;
2402         int err;
2403         int i;
2404
2405         /* Create instance with 4 ports */
2406         if (init(t, 2, 3) < 0 ||
2407                         create_ports(t, 3) < 0 ||
2408                         create_atomic_qids(t, 2) < 0) {
2409                 printf("%d: Error initializing device\n", __LINE__);
2410                 return -1;
2411         }
2412
2413         /* CQ mapping to QID */
2414         err = rte_event_port_link(evdev, t->port[p1], &t->qid[0], NULL, 1);
2415         if (err != 1) {
2416                 printf("%d: error mapping lb qid\n", __LINE__);
2417                 cleanup(t);
2418                 return -1;
2419         }
2420         err = rte_event_port_link(evdev, t->port[p2], &t->qid[1], NULL, 1);
2421         if (err != 1) {
2422                 printf("%d: error mapping lb qid\n", __LINE__);
2423                 cleanup(t);
2424                 return -1;
2425         }
2426
2427         if (rte_event_dev_start(evdev) < 0) {
2428                 printf("%d: Error with start call\n", __LINE__);
2429                 return -1;
2430         }
2431
2432         /************** FORWARD ****************/
2433 #define QID1_NUM 5
2434         for (i = 0; i < QID1_NUM; i++) {
2435                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2436
2437                 if (!arp) {
2438                         printf("%d: gen of pkt failed\n", __LINE__);
2439                         goto err;
2440                 }
2441
2442                 ev.queue_id =  t->qid[0];
2443                 ev.op = RTE_EVENT_OP_NEW;
2444                 ev.mbuf = arp;
2445                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2446                 if (err != 1) {
2447                         printf("%d: Failed to enqueue\n", __LINE__);
2448                         goto err;
2449                 }
2450         }
2451 #define QID2_NUM 3
2452         for (i = 0; i < QID2_NUM; i++) {
2453                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2454
2455                 if (!arp) {
2456                         printf("%d: gen of pkt failed\n", __LINE__);
2457                         goto err;
2458                 }
2459                 ev.queue_id =  t->qid[1];
2460                 ev.op = RTE_EVENT_OP_NEW;
2461                 ev.mbuf = arp;
2462                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2463                 if (err != 1) {
2464                         printf("%d: Failed to enqueue\n", __LINE__);
2465                         goto err;
2466                 }
2467         }
2468
2469         /* schedule */
2470         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2471
2472         err = test_event_dev_stats_get(evdev, &stats);
2473         if (err) {
2474                 printf("%d: failed to get stats\n", __LINE__);
2475                 goto err;
2476         }
2477
2478         if (stats.rx_pkts != QID1_NUM + QID2_NUM ||
2479                         stats.tx_pkts != QID1_NUM + QID2_NUM) {
2480                 printf("%d: Sched core didn't handle pkt as expected\n",
2481                                 __LINE__);
2482                 goto err;
2483         }
2484
2485         if (stats.port_inflight[p1] != QID1_NUM) {
2486                 printf("%d: %s port 1 inflight not correct\n", __LINE__,
2487                                 __func__);
2488                 goto err;
2489         }
2490         if (stats.port_inflight[p2] != QID2_NUM) {
2491                 printf("%d: %s port 2 inflight not correct\n", __LINE__,
2492                                 __func__);
2493                 goto err;
2494         }
2495
2496         /************** DEQUEUE INFLIGHT COUNT CHECKS  ****************/
2497         /* port 1 */
2498         struct rte_event events[QID1_NUM + QID2_NUM];
2499         uint32_t deq_pkts = rte_event_dequeue_burst(evdev, t->port[p1], events,
2500                         RTE_DIM(events), 0);
2501
2502         if (deq_pkts != QID1_NUM) {
2503                 printf("%d: Port 1: DEQUEUE inflight failed\n", __LINE__);
2504                 goto err;
2505         }
2506         err = test_event_dev_stats_get(evdev, &stats);
2507         if (stats.port_inflight[p1] != QID1_NUM) {
2508                 printf("%d: port 1 inflight decrement after DEQ != 0\n",
2509                                 __LINE__);
2510                 goto err;
2511         }
2512         for (i = 0; i < QID1_NUM; i++) {
2513                 err = rte_event_enqueue_burst(evdev, t->port[p1], &release_ev,
2514                                 1);
2515                 if (err != 1) {
2516                         printf("%d: %s rte enqueue of inf release failed\n",
2517                                 __LINE__, __func__);
2518                         goto err;
2519                 }
2520         }
2521
2522         /*
2523          * As the scheduler core decrements inflights, it needs to run to
2524          * process packets to act on the drop messages
2525          */
2526         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2527
2528         err = test_event_dev_stats_get(evdev, &stats);
2529         if (stats.port_inflight[p1] != 0) {
2530                 printf("%d: port 1 inflight NON NULL after DROP\n", __LINE__);
2531                 goto err;
2532         }
2533
2534         /* port2 */
2535         deq_pkts = rte_event_dequeue_burst(evdev, t->port[p2], events,
2536                         RTE_DIM(events), 0);
2537         if (deq_pkts != QID2_NUM) {
2538                 printf("%d: Port 2: DEQUEUE inflight failed\n", __LINE__);
2539                 goto err;
2540         }
2541         err = test_event_dev_stats_get(evdev, &stats);
2542         if (stats.port_inflight[p2] != QID2_NUM) {
2543                 printf("%d: port 1 inflight decrement after DEQ != 0\n",
2544                                 __LINE__);
2545                 goto err;
2546         }
2547         for (i = 0; i < QID2_NUM; i++) {
2548                 err = rte_event_enqueue_burst(evdev, t->port[p2], &release_ev,
2549                                 1);
2550                 if (err != 1) {
2551                         printf("%d: %s rte enqueue of inf release failed\n",
2552                                 __LINE__, __func__);
2553                         goto err;
2554                 }
2555         }
2556
2557         /*
2558          * As the scheduler core decrements inflights, it needs to run to
2559          * process packets to act on the drop messages
2560          */
2561         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2562
2563         err = test_event_dev_stats_get(evdev, &stats);
2564         if (stats.port_inflight[p2] != 0) {
2565                 printf("%d: port 2 inflight NON NULL after DROP\n", __LINE__);
2566                 goto err;
2567         }
2568         cleanup(t);
2569         return 0;
2570
2571 err:
2572         rte_event_dev_dump(evdev, stdout);
2573         cleanup(t);
2574         return -1;
2575 }
2576
2577 static int
2578 parallel_basic(struct test *t, int check_order)
2579 {
2580         const uint8_t rx_port = 0;
2581         const uint8_t w1_port = 1;
2582         const uint8_t w3_port = 3;
2583         const uint8_t tx_port = 4;
2584         int err;
2585         int i;
2586         uint32_t deq_pkts, j;
2587         struct rte_mbuf *mbufs[3];
2588         struct rte_mbuf *mbufs_out[3] = { 0 };
2589         const uint32_t MAGIC_SEQN = 1234;
2590
2591         /* Create instance with 4 ports */
2592         if (init(t, 2, tx_port + 1) < 0 ||
2593                         create_ports(t, tx_port + 1) < 0 ||
2594                         (check_order ?  create_ordered_qids(t, 1) :
2595                                 create_unordered_qids(t, 1)) < 0 ||
2596                         create_directed_qids(t, 1, &tx_port)) {
2597                 printf("%d: Error initializing device\n", __LINE__);
2598                 return -1;
2599         }
2600
2601         /*
2602          * CQ mapping to QID
2603          * We need three ports, all mapped to the same ordered qid0. Then we'll
2604          * take a packet out to each port, re-enqueue in reverse order,
2605          * then make sure the reordering has taken place properly when we
2606          * dequeue from the tx_port.
2607          *
2608          * Simplified test setup diagram:
2609          *
2610          * rx_port        w1_port
2611          *        \     /         \
2612          *         qid0 - w2_port - qid1
2613          *              \         /     \
2614          *                w3_port        tx_port
2615          */
2616         /* CQ mapping to QID for LB ports (directed mapped on create) */
2617         for (i = w1_port; i <= w3_port; i++) {
2618                 err = rte_event_port_link(evdev, t->port[i], &t->qid[0], NULL,
2619                                 1);
2620                 if (err != 1) {
2621                         printf("%d: error mapping lb qid\n", __LINE__);
2622                         cleanup(t);
2623                         return -1;
2624                 }
2625         }
2626
2627         if (rte_event_dev_start(evdev) < 0) {
2628                 printf("%d: Error with start call\n", __LINE__);
2629                 return -1;
2630         }
2631
2632         /* Enqueue 3 packets to the rx port */
2633         for (i = 0; i < 3; i++) {
2634                 struct rte_event ev;
2635                 mbufs[i] = rte_gen_arp(0, t->mbuf_pool);
2636                 if (!mbufs[i]) {
2637                         printf("%d: gen of pkt failed\n", __LINE__);
2638                         return -1;
2639                 }
2640
2641                 ev.queue_id = t->qid[0];
2642                 ev.op = RTE_EVENT_OP_NEW;
2643                 ev.mbuf = mbufs[i];
2644                 mbufs[i]->seqn = MAGIC_SEQN + i;
2645
2646                 /* generate pkt and enqueue */
2647                 err = rte_event_enqueue_burst(evdev, t->port[rx_port], &ev, 1);
2648                 if (err != 1) {
2649                         printf("%d: Failed to enqueue pkt %u, retval = %u\n",
2650                                         __LINE__, i, err);
2651                         return -1;
2652                 }
2653         }
2654
2655         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2656
2657         /* use extra slot to make logic in loops easier */
2658         struct rte_event deq_ev[w3_port + 1];
2659
2660         /* Dequeue the 3 packets, one from each worker port */
2661         for (i = w1_port; i <= w3_port; i++) {
2662                 deq_pkts = rte_event_dequeue_burst(evdev, t->port[i],
2663                                 &deq_ev[i], 1, 0);
2664                 if (deq_pkts != 1) {
2665                         printf("%d: Failed to deq\n", __LINE__);
2666                         rte_event_dev_dump(evdev, stdout);
2667                         return -1;
2668                 }
2669         }
2670
2671         /* Enqueue each packet in reverse order, flushing after each one */
2672         for (i = w3_port; i >= w1_port; i--) {
2673
2674                 deq_ev[i].op = RTE_EVENT_OP_FORWARD;
2675                 deq_ev[i].queue_id = t->qid[1];
2676                 err = rte_event_enqueue_burst(evdev, t->port[i], &deq_ev[i], 1);
2677                 if (err != 1) {
2678                         printf("%d: Failed to enqueue\n", __LINE__);
2679                         return -1;
2680                 }
2681         }
2682         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2683
2684         /* dequeue from the tx ports, we should get 3 packets */
2685         deq_pkts = rte_event_dequeue_burst(evdev, t->port[tx_port], deq_ev,
2686                         3, 0);
2687
2688         /* Check to see if we've got all 3 packets */
2689         if (deq_pkts != 3) {
2690                 printf("%d: expected 3 pkts at tx port got %d from port %d\n",
2691                         __LINE__, deq_pkts, tx_port);
2692                 rte_event_dev_dump(evdev, stdout);
2693                 return 1;
2694         }
2695
2696         /* Check to see if the sequence numbers are in expected order */
2697         if (check_order) {
2698                 for (j = 0 ; j < deq_pkts ; j++) {
2699                         if (deq_ev[j].mbuf->seqn != MAGIC_SEQN + j) {
2700                                 printf(
2701                                         "%d: Incorrect sequence number(%d) from port %d\n",
2702                                         __LINE__, mbufs_out[j]->seqn, tx_port);
2703                                 return -1;
2704                         }
2705                 }
2706         }
2707
2708         /* Destroy the instance */
2709         cleanup(t);
2710         return 0;
2711 }
2712
2713 static int
2714 ordered_basic(struct test *t)
2715 {
2716         return parallel_basic(t, 1);
2717 }
2718
2719 static int
2720 unordered_basic(struct test *t)
2721 {
2722         return parallel_basic(t, 0);
2723 }
2724
2725 static int
2726 holb(struct test *t) /* test to check we avoid basic head-of-line blocking */
2727 {
2728         const struct rte_event new_ev = {
2729                         .op = RTE_EVENT_OP_NEW
2730                         /* all other fields zero */
2731         };
2732         struct rte_event ev = new_ev;
2733         unsigned int rx_port = 0; /* port we get the first flow on */
2734         char rx_port_used_stat[64];
2735         char rx_port_free_stat[64];
2736         char other_port_used_stat[64];
2737
2738         if (init(t, 1, 2) < 0 ||
2739                         create_ports(t, 2) < 0 ||
2740                         create_atomic_qids(t, 1) < 0) {
2741                 printf("%d: Error initializing device\n", __LINE__);
2742                 return -1;
2743         }
2744         int nb_links = rte_event_port_link(evdev, t->port[1], NULL, NULL, 0);
2745         if (rte_event_port_link(evdev, t->port[0], NULL, NULL, 0) != 1 ||
2746                         nb_links != 1) {
2747                 printf("%d: Error links queue to ports\n", __LINE__);
2748                 goto err;
2749         }
2750         if (rte_event_dev_start(evdev) < 0) {
2751                 printf("%d: Error with start call\n", __LINE__);
2752                 goto err;
2753         }
2754
2755         /* send one packet and see where it goes, port 0 or 1 */
2756         if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2757                 printf("%d: Error doing first enqueue\n", __LINE__);
2758                 goto err;
2759         }
2760         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2761
2762         if (rte_event_dev_xstats_by_name_get(evdev, "port_0_cq_ring_used", NULL)
2763                         != 1)
2764                 rx_port = 1;
2765
2766         snprintf(rx_port_used_stat, sizeof(rx_port_used_stat),
2767                         "port_%u_cq_ring_used", rx_port);
2768         snprintf(rx_port_free_stat, sizeof(rx_port_free_stat),
2769                         "port_%u_cq_ring_free", rx_port);
2770         snprintf(other_port_used_stat, sizeof(other_port_used_stat),
2771                         "port_%u_cq_ring_used", rx_port ^ 1);
2772         if (rte_event_dev_xstats_by_name_get(evdev, rx_port_used_stat, NULL)
2773                         != 1) {
2774                 printf("%d: Error, first event not scheduled\n", __LINE__);
2775                 goto err;
2776         }
2777
2778         /* now fill up the rx port's queue with one flow to cause HOLB */
2779         do {
2780                 ev = new_ev;
2781                 if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2782                         printf("%d: Error with enqueue\n", __LINE__);
2783                         goto err;
2784                 }
2785                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
2786         } while (rte_event_dev_xstats_by_name_get(evdev,
2787                                 rx_port_free_stat, NULL) != 0);
2788
2789         /* one more packet, which needs to stay in IQ - i.e. HOLB */
2790         ev = new_ev;
2791         if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2792                 printf("%d: Error with enqueue\n", __LINE__);
2793                 goto err;
2794         }
2795         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2796
2797         /* check that the other port still has an empty CQ */
2798         if (rte_event_dev_xstats_by_name_get(evdev, other_port_used_stat, NULL)
2799                         != 0) {
2800                 printf("%d: Error, second port CQ is not empty\n", __LINE__);
2801                 goto err;
2802         }
2803         /* check IQ now has one packet */
2804         if (rte_event_dev_xstats_by_name_get(evdev, "qid_0_iq_0_used", NULL)
2805                         != 1) {
2806                 printf("%d: Error, QID does not have exactly 1 packet\n",
2807                         __LINE__);
2808                 goto err;
2809         }
2810
2811         /* send another flow, which should pass the other IQ entry */
2812         ev = new_ev;
2813         ev.flow_id = 1;
2814         if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2815                 printf("%d: Error with enqueue\n", __LINE__);
2816                 goto err;
2817         }
2818         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2819
2820         if (rte_event_dev_xstats_by_name_get(evdev, other_port_used_stat, NULL)
2821                         != 1) {
2822                 printf("%d: Error, second flow did not pass out first\n",
2823                         __LINE__);
2824                 goto err;
2825         }
2826
2827         if (rte_event_dev_xstats_by_name_get(evdev, "qid_0_iq_0_used", NULL)
2828                         != 1) {
2829                 printf("%d: Error, QID does not have exactly 1 packet\n",
2830                         __LINE__);
2831                 goto err;
2832         }
2833         cleanup(t);
2834         return 0;
2835 err:
2836         rte_event_dev_dump(evdev, stdout);
2837         cleanup(t);
2838         return -1;
2839 }
2840
2841 static int
2842 worker_loopback_worker_fn(void *arg)
2843 {
2844         struct test *t = arg;
2845         uint8_t port = t->port[1];
2846         int count = 0;
2847         int enqd;
2848
2849         /*
2850          * Takes packets from the input port and then loops them back through
2851          * the Eventdev. Each packet gets looped through QIDs 0-8, 16 times
2852          * so each packet goes through 8*16 = 128 times.
2853          */
2854         printf("%d: \tWorker function started\n", __LINE__);
2855         while (count < NUM_PACKETS) {
2856 #define BURST_SIZE 32
2857                 struct rte_event ev[BURST_SIZE];
2858                 uint16_t i, nb_rx = rte_event_dequeue_burst(evdev, port, ev,
2859                                 BURST_SIZE, 0);
2860                 if (nb_rx == 0) {
2861                         rte_pause();
2862                         continue;
2863                 }
2864
2865                 for (i = 0; i < nb_rx; i++) {
2866                         ev[i].queue_id++;
2867                         if (ev[i].queue_id != 8) {
2868                                 ev[i].op = RTE_EVENT_OP_FORWARD;
2869                                 enqd = rte_event_enqueue_burst(evdev, port,
2870                                                 &ev[i], 1);
2871                                 if (enqd != 1) {
2872                                         printf("%d: Can't enqueue FWD!!\n",
2873                                                         __LINE__);
2874                                         return -1;
2875                                 }
2876                                 continue;
2877                         }
2878
2879                         ev[i].queue_id = 0;
2880                         ev[i].mbuf->udata64++;
2881                         if (ev[i].mbuf->udata64 != 16) {
2882                                 ev[i].op = RTE_EVENT_OP_FORWARD;
2883                                 enqd = rte_event_enqueue_burst(evdev, port,
2884                                                 &ev[i], 1);
2885                                 if (enqd != 1) {
2886                                         printf("%d: Can't enqueue FWD!!\n",
2887                                                         __LINE__);
2888                                         return -1;
2889                                 }
2890                                 continue;
2891                         }
2892                         /* we have hit 16 iterations through system - drop */
2893                         rte_pktmbuf_free(ev[i].mbuf);
2894                         count++;
2895                         ev[i].op = RTE_EVENT_OP_RELEASE;
2896                         enqd = rte_event_enqueue_burst(evdev, port, &ev[i], 1);
2897                         if (enqd != 1) {
2898                                 printf("%d drop enqueue failed\n", __LINE__);
2899                                 return -1;
2900                         }
2901                 }
2902         }
2903
2904         return 0;
2905 }
2906
2907 static int
2908 worker_loopback_producer_fn(void *arg)
2909 {
2910         struct test *t = arg;
2911         uint8_t port = t->port[0];
2912         uint64_t count = 0;
2913
2914         printf("%d: \tProducer function started\n", __LINE__);
2915         while (count < NUM_PACKETS) {
2916                 struct rte_mbuf *m = 0;
2917                 do {
2918                         m = rte_pktmbuf_alloc(t->mbuf_pool);
2919                 } while (m == NULL);
2920
2921                 m->udata64 = 0;
2922
2923                 struct rte_event ev = {
2924                                 .op = RTE_EVENT_OP_NEW,
2925                                 .queue_id = t->qid[0],
2926                                 .flow_id = (uintptr_t)m & 0xFFFF,
2927                                 .mbuf = m,
2928                 };
2929
2930                 if (rte_event_enqueue_burst(evdev, port, &ev, 1) != 1) {
2931                         while (rte_event_enqueue_burst(evdev, port, &ev, 1) !=
2932                                         1)
2933                                 rte_pause();
2934                 }
2935
2936                 count++;
2937         }
2938
2939         return 0;
2940 }
2941
2942 static int
2943 worker_loopback(struct test *t)
2944 {
2945         /* use a single producer core, and a worker core to see what happens
2946          * if the worker loops packets back multiple times
2947          */
2948         struct test_event_dev_stats stats;
2949         uint64_t print_cycles = 0, cycles = 0;
2950         uint64_t tx_pkts = 0;
2951         int err;
2952         int w_lcore, p_lcore;
2953
2954         if (init(t, 8, 2) < 0 ||
2955                         create_atomic_qids(t, 8) < 0) {
2956                 printf("%d: Error initializing device\n", __LINE__);
2957                 return -1;
2958         }
2959
2960         /* RX with low max events */
2961         static struct rte_event_port_conf conf = {
2962                         .dequeue_depth = 32,
2963                         .enqueue_depth = 64,
2964         };
2965         /* beware: this cannot be initialized in the static above as it would
2966          * only be initialized once - and this needs to be set for multiple runs
2967          */
2968         conf.new_event_threshold = 512;
2969
2970         if (rte_event_port_setup(evdev, 0, &conf) < 0) {
2971                 printf("Error setting up RX port\n");
2972                 return -1;
2973         }
2974         t->port[0] = 0;
2975         /* TX with higher max events */
2976         conf.new_event_threshold = 4096;
2977         if (rte_event_port_setup(evdev, 1, &conf) < 0) {
2978                 printf("Error setting up TX port\n");
2979                 return -1;
2980         }
2981         t->port[1] = 1;
2982
2983         /* CQ mapping to QID */
2984         err = rte_event_port_link(evdev, t->port[1], NULL, NULL, 0);
2985         if (err != 8) { /* should have mapped all queues*/
2986                 printf("%d: error mapping port 2 to all qids\n", __LINE__);
2987                 return -1;
2988         }
2989
2990         if (rte_event_dev_start(evdev) < 0) {
2991                 printf("%d: Error with start call\n", __LINE__);
2992                 return -1;
2993         }
2994
2995         p_lcore = rte_get_next_lcore(
2996                         /* start core */ -1,
2997                         /* skip master */ 1,
2998                         /* wrap */ 0);
2999         w_lcore = rte_get_next_lcore(p_lcore, 1, 0);
3000
3001         rte_eal_remote_launch(worker_loopback_producer_fn, t, p_lcore);
3002         rte_eal_remote_launch(worker_loopback_worker_fn, t, w_lcore);
3003
3004         print_cycles = cycles = rte_get_timer_cycles();
3005         while (rte_eal_get_lcore_state(p_lcore) != FINISHED ||
3006                         rte_eal_get_lcore_state(w_lcore) != FINISHED) {
3007
3008                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
3009
3010                 uint64_t new_cycles = rte_get_timer_cycles();
3011
3012                 if (new_cycles - print_cycles > rte_get_timer_hz()) {
3013                         test_event_dev_stats_get(evdev, &stats);
3014                         printf(
3015                                 "%d: \tSched Rx = %"PRIu64", Tx = %"PRIu64"\n",
3016                                 __LINE__, stats.rx_pkts, stats.tx_pkts);
3017
3018                         print_cycles = new_cycles;
3019                 }
3020                 if (new_cycles - cycles > rte_get_timer_hz() * 3) {
3021                         test_event_dev_stats_get(evdev, &stats);
3022                         if (stats.tx_pkts == tx_pkts) {
3023                                 rte_event_dev_dump(evdev, stdout);
3024                                 printf("Dumping xstats:\n");
3025                                 xstats_print();
3026                                 printf(
3027                                         "%d: No schedules for seconds, deadlock\n",
3028                                         __LINE__);
3029                                 return -1;
3030                         }
3031                         tx_pkts = stats.tx_pkts;
3032                         cycles = new_cycles;
3033                 }
3034         }
3035         rte_service_run_iter_on_app_lcore(t->service_id, 1);
3036         /* ensure all completions are flushed */
3037
3038         rte_eal_mp_wait_lcore();
3039
3040         cleanup(t);
3041         return 0;
3042 }
3043
3044 static struct rte_mempool *eventdev_func_mempool;
3045
3046 static int
3047 test_sw_eventdev(void)
3048 {
3049         struct test *t = malloc(sizeof(struct test));
3050         int ret;
3051
3052         /* manually initialize the op, older gcc's complain on static
3053          * initialization of struct elements that are a bitfield.
3054          */
3055         release_ev.op = RTE_EVENT_OP_RELEASE;
3056
3057         const char *eventdev_name = "event_sw0";
3058         evdev = rte_event_dev_get_dev_id(eventdev_name);
3059         if (evdev < 0) {
3060                 printf("%d: Eventdev %s not found - creating.\n",
3061                                 __LINE__, eventdev_name);
3062                 if (rte_vdev_init(eventdev_name, NULL) < 0) {
3063                         printf("Error creating eventdev\n");
3064                         return -1;
3065                 }
3066                 evdev = rte_event_dev_get_dev_id(eventdev_name);
3067                 if (evdev < 0) {
3068                         printf("Error finding newly created eventdev\n");
3069                         return -1;
3070                 }
3071         }
3072
3073         if (rte_event_dev_service_id_get(evdev, &t->service_id) < 0) {
3074                 printf("Failed to get service ID for software event dev\n");
3075                 return -1;
3076         }
3077
3078         rte_service_runstate_set(t->service_id, 1);
3079         rte_service_set_runstate_mapped_check(t->service_id, 0);
3080
3081         /* Only create mbuf pool once, reuse for each test run */
3082         if (!eventdev_func_mempool) {
3083                 eventdev_func_mempool = rte_pktmbuf_pool_create(
3084                                 "EVENTDEV_SW_SA_MBUF_POOL",
3085                                 (1<<12), /* 4k buffers */
3086                                 32 /*MBUF_CACHE_SIZE*/,
3087                                 0,
3088                                 512, /* use very small mbufs */
3089                                 rte_socket_id());
3090                 if (!eventdev_func_mempool) {
3091                         printf("ERROR creating mempool\n");
3092                         return -1;
3093                 }
3094         }
3095         t->mbuf_pool = eventdev_func_mempool;
3096         printf("*** Running Single Directed Packet test...\n");
3097         ret = test_single_directed_packet(t);
3098         if (ret != 0) {
3099                 printf("ERROR - Single Directed Packet test FAILED.\n");
3100                 return ret;
3101         }
3102         printf("*** Running Directed Forward Credit test...\n");
3103         ret = test_directed_forward_credits(t);
3104         if (ret != 0) {
3105                 printf("ERROR - Directed Forward Credit test FAILED.\n");
3106                 return ret;
3107         }
3108         printf("*** Running Single Load Balanced Packet test...\n");
3109         ret = single_packet(t);
3110         if (ret != 0) {
3111                 printf("ERROR - Single Packet test FAILED.\n");
3112                 return ret;
3113         }
3114         printf("*** Running Unordered Basic test...\n");
3115         ret = unordered_basic(t);
3116         if (ret != 0) {
3117                 printf("ERROR -  Unordered Basic test FAILED.\n");
3118                 return ret;
3119         }
3120         printf("*** Running Ordered Basic test...\n");
3121         ret = ordered_basic(t);
3122         if (ret != 0) {
3123                 printf("ERROR -  Ordered Basic test FAILED.\n");
3124                 return ret;
3125         }
3126         printf("*** Running Burst Packets test...\n");
3127         ret = burst_packets(t);
3128         if (ret != 0) {
3129                 printf("ERROR - Burst Packets test FAILED.\n");
3130                 return ret;
3131         }
3132         printf("*** Running Load Balancing test...\n");
3133         ret = load_balancing(t);
3134         if (ret != 0) {
3135                 printf("ERROR - Load Balancing test FAILED.\n");
3136                 return ret;
3137         }
3138         printf("*** Running Prioritized Directed test...\n");
3139         ret = test_priority_directed(t);
3140         if (ret != 0) {
3141                 printf("ERROR - Prioritized Directed test FAILED.\n");
3142                 return ret;
3143         }
3144         printf("*** Running Prioritized Atomic test...\n");
3145         ret = test_priority_atomic(t);
3146         if (ret != 0) {
3147                 printf("ERROR - Prioritized Atomic test FAILED.\n");
3148                 return ret;
3149         }
3150
3151         printf("*** Running Prioritized Ordered test...\n");
3152         ret = test_priority_ordered(t);
3153         if (ret != 0) {
3154                 printf("ERROR - Prioritized Ordered test FAILED.\n");
3155                 return ret;
3156         }
3157         printf("*** Running Prioritized Unordered test...\n");
3158         ret = test_priority_unordered(t);
3159         if (ret != 0) {
3160                 printf("ERROR - Prioritized Unordered test FAILED.\n");
3161                 return ret;
3162         }
3163         printf("*** Running Invalid QID test...\n");
3164         ret = invalid_qid(t);
3165         if (ret != 0) {
3166                 printf("ERROR - Invalid QID test FAILED.\n");
3167                 return ret;
3168         }
3169         printf("*** Running Load Balancing History test...\n");
3170         ret = load_balancing_history(t);
3171         if (ret != 0) {
3172                 printf("ERROR - Load Balancing History test FAILED.\n");
3173                 return ret;
3174         }
3175         printf("*** Running Inflight Count test...\n");
3176         ret = inflight_counts(t);
3177         if (ret != 0) {
3178                 printf("ERROR - Inflight Count test FAILED.\n");
3179                 return ret;
3180         }
3181         printf("*** Running Abuse Inflights test...\n");
3182         ret = abuse_inflights(t);
3183         if (ret != 0) {
3184                 printf("ERROR - Abuse Inflights test FAILED.\n");
3185                 return ret;
3186         }
3187         printf("*** Running XStats test...\n");
3188         ret = xstats_tests(t);
3189         if (ret != 0) {
3190                 printf("ERROR - XStats test FAILED.\n");
3191                 return ret;
3192         }
3193         printf("*** Running XStats ID Reset test...\n");
3194         ret = xstats_id_reset_tests(t);
3195         if (ret != 0) {
3196                 printf("ERROR - XStats ID Reset test FAILED.\n");
3197                 return ret;
3198         }
3199         printf("*** Running XStats Brute Force test...\n");
3200         ret = xstats_brute_force(t);
3201         if (ret != 0) {
3202                 printf("ERROR - XStats Brute Force test FAILED.\n");
3203                 return ret;
3204         }
3205         printf("*** Running XStats ID Abuse test...\n");
3206         ret = xstats_id_abuse_tests(t);
3207         if (ret != 0) {
3208                 printf("ERROR - XStats ID Abuse test FAILED.\n");
3209                 return ret;
3210         }
3211         printf("*** Running QID Priority test...\n");
3212         ret = qid_priorities(t);
3213         if (ret != 0) {
3214                 printf("ERROR - QID Priority test FAILED.\n");
3215                 return ret;
3216         }
3217         printf("*** Running Ordered Reconfigure test...\n");
3218         ret = ordered_reconfigure(t);
3219         if (ret != 0) {
3220                 printf("ERROR - Ordered Reconfigure test FAILED.\n");
3221                 return ret;
3222         }
3223         printf("*** Running Port LB Single Reconfig test...\n");
3224         ret = port_single_lb_reconfig(t);
3225         if (ret != 0) {
3226                 printf("ERROR - Port LB Single Reconfig test FAILED.\n");
3227                 return ret;
3228         }
3229         printf("*** Running Port Reconfig Credits test...\n");
3230         ret = port_reconfig_credits(t);
3231         if (ret != 0) {
3232                 printf("ERROR - Port Reconfig Credits Reset test FAILED.\n");
3233                 return ret;
3234         }
3235         printf("*** Running Head-of-line-blocking test...\n");
3236         ret = holb(t);
3237         if (ret != 0) {
3238                 printf("ERROR - Head-of-line-blocking test FAILED.\n");
3239                 return ret;
3240         }
3241         if (rte_lcore_count() >= 3) {
3242                 printf("*** Running Worker loopback test...\n");
3243                 ret = worker_loopback(t);
3244                 if (ret != 0) {
3245                         printf("ERROR - Worker loopback test FAILED.\n");
3246                         return ret;
3247                 }
3248         } else {
3249                 printf("### Not enough cores for worker loopback test.\n");
3250                 printf("### Need at least 3 cores for test.\n");
3251         }
3252         /*
3253          * Free test instance, leaving mempool initialized, and a pointer to it
3254          * in static eventdev_func_mempool, as it is re-used on re-runs
3255          */
3256         free(t);
3257
3258         return 0;
3259 }
3260
3261 REGISTER_TEST_COMMAND(eventdev_sw_autotest, test_sw_eventdev);