test/test/test_eventdev_sw.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <string.h>
   7 #include <stdint.h>
   8 #include <errno.h>
   9 #include <unistd.h>
  10 #include <sys/queue.h>
  11
  12 #include <rte_memory.h>
  13 #include <rte_launch.h>
  14 #include <rte_eal.h>
  15 #include <rte_per_lcore.h>
  16 #include <rte_lcore.h>
  17 #include <rte_debug.h>
  18 #include <rte_ethdev.h>
  19 #include <rte_cycles.h>
  20 #include <rte_eventdev.h>
  21 #include <rte_pause.h>
  22 #include <rte_service.h>
  23 #include <rte_service_component.h>
  24 #include <rte_bus_vdev.h>
  25
  26 #include "test.h"
  27
  28 #define MAX_PORTS 16
  29 #define MAX_QIDS 16
  30 #define NUM_PACKETS (1<<18)
  31
  32 static int evdev;
  33
  34 struct test {
  35         struct rte_mempool *mbuf_pool;
  36         uint8_t port[MAX_PORTS];
  37         uint8_t qid[MAX_QIDS];
  38         int nb_qids;
  39         uint32_t service_id;
  40 };
  41
  42 static struct rte_event release_ev;
  43
  44 static inline struct rte_mbuf *
  45 rte_gen_arp(int portid, struct rte_mempool *mp)
  46 {
  47         /*
  48          * len = 14 + 46
  49          * ARP, Request who-has 10.0.0.1 tell 10.0.0.2, length 46
  50          */
  51         static const uint8_t arp_request[] = {
  52                 /*0x0000:*/ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xec, 0xa8,
  53                 0x6b, 0xfd, 0x02, 0x29, 0x08, 0x06, 0x00, 0x01,
  54                 /*0x0010:*/ 0x08, 0x00, 0x06, 0x04, 0x00, 0x01, 0xec, 0xa8,
  55                 0x6b, 0xfd, 0x02, 0x29, 0x0a, 0x00, 0x00, 0x01,
  56                 /*0x0020:*/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00,
  57                 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  58                 /*0x0030:*/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  59                 0x00, 0x00, 0x00, 0x00
  60         };
  61         struct rte_mbuf *m;
  62         int pkt_len = sizeof(arp_request) - 1;
  63
  64         m = rte_pktmbuf_alloc(mp);
  65         if (!m)
  66                 return 0;
  67
  68         memcpy((void *)((uintptr_t)m->buf_addr + m->data_off),
  69                 arp_request, pkt_len);
  70         rte_pktmbuf_pkt_len(m) = pkt_len;
  71         rte_pktmbuf_data_len(m) = pkt_len;
  72
  73         RTE_SET_USED(portid);
  74
  75         return m;
  76 }
  77
  78 static void
  79 xstats_print(void)
  80 {
  81         const uint32_t XSTATS_MAX = 1024;
  82         uint32_t i;
  83         uint32_t ids[XSTATS_MAX];
  84         uint64_t values[XSTATS_MAX];
  85         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
  86
  87         for (i = 0; i < XSTATS_MAX; i++)
  88                 ids[i] = i;
  89
  90         /* Device names / values */
  91         int ret = rte_event_dev_xstats_names_get(evdev,
  92                                         RTE_EVENT_DEV_XSTATS_DEVICE, 0,
  93                                         xstats_names, ids, XSTATS_MAX);
  94         if (ret < 0) {
  95                 printf("%d: xstats names get() returned error\n",
  96                         __LINE__);
  97                 return;
  98         }
  99         ret = rte_event_dev_xstats_get(evdev,
 100                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 101                                         0, ids, values, ret);
 102         if (ret > (signed int)XSTATS_MAX)
 103                 printf("%s %d: more xstats available than space\n",
 104                                 __func__, __LINE__);
 105         for (i = 0; (signed int)i < ret; i++) {
 106                 printf("%d : %s : %"PRIu64"\n",
 107                                 i, xstats_names[i].name, values[i]);
 108         }
 109
 110         /* Port names / values */
 111         ret = rte_event_dev_xstats_names_get(evdev,
 112                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
 113                                         xstats_names, ids, XSTATS_MAX);
 114         ret = rte_event_dev_xstats_get(evdev,
 115                                         RTE_EVENT_DEV_XSTATS_PORT, 1,
 116                                         ids, values, ret);
 117         if (ret > (signed int)XSTATS_MAX)
 118                 printf("%s %d: more xstats available than space\n",
 119                                 __func__, __LINE__);
 120         for (i = 0; (signed int)i < ret; i++) {
 121                 printf("%d : %s : %"PRIu64"\n",
 122                                 i, xstats_names[i].name, values[i]);
 123         }
 124
 125         /* Queue names / values */
 126         ret = rte_event_dev_xstats_names_get(evdev,
 127                                         RTE_EVENT_DEV_XSTATS_QUEUE, 0,
 128                                         xstats_names, ids, XSTATS_MAX);
 129         ret = rte_event_dev_xstats_get(evdev,
 130                                         RTE_EVENT_DEV_XSTATS_QUEUE,
 131                                         1, ids, values, ret);
 132         if (ret > (signed int)XSTATS_MAX)
 133                 printf("%s %d: more xstats available than space\n",
 134                                 __func__, __LINE__);
 135         for (i = 0; (signed int)i < ret; i++) {
 136                 printf("%d : %s : %"PRIu64"\n",
 137                                 i, xstats_names[i].name, values[i]);
 138         }
 139 }
 140
 141 /* initialization and config */
 142 static inline int
 143 init(struct test *t, int nb_queues, int nb_ports)
 144 {
 145         struct rte_event_dev_config config = {
 146                         .nb_event_queues = nb_queues,
 147                         .nb_event_ports = nb_ports,
 148                         .nb_event_queue_flows = 1024,
 149                         .nb_events_limit = 4096,
 150                         .nb_event_port_dequeue_depth = 128,
 151                         .nb_event_port_enqueue_depth = 128,
 152         };
 153         int ret;
 154
 155         void *temp = t->mbuf_pool; /* save and restore mbuf pool */
 156
 157         memset(t, 0, sizeof(*t));
 158         t->mbuf_pool = temp;
 159
 160         ret = rte_event_dev_configure(evdev, &config);
 161         if (ret < 0)
 162                 printf("%d: Error configuring device\n", __LINE__);
 163         return ret;
 164 };
 165
 166 static inline int
 167 create_ports(struct test *t, int num_ports)
 168 {
 169         int i;
 170         static const struct rte_event_port_conf conf = {
 171                         .new_event_threshold = 1024,
 172                         .dequeue_depth = 32,
 173                         .enqueue_depth = 64,
 174         };
 175         if (num_ports > MAX_PORTS)
 176                 return -1;
 177
 178         for (i = 0; i < num_ports; i++) {
 179                 if (rte_event_port_setup(evdev, i, &conf) < 0) {
 180                         printf("Error setting up port %d\n", i);
 181                         return -1;
 182                 }
 183                 t->port[i] = i;
 184         }
 185
 186         return 0;
 187 }
 188
 189 static inline int
 190 create_lb_qids(struct test *t, int num_qids, uint32_t flags)
 191 {
 192         int i;
 193
 194         /* Q creation */
 195         const struct rte_event_queue_conf conf = {
 196                         .schedule_type = flags,
 197                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 198                         .nb_atomic_flows = 1024,
 199                         .nb_atomic_order_sequences = 1024,
 200         };
 201
 202         for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
 203                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
 204                         printf("%d: error creating qid %d\n", __LINE__, i);
 205                         return -1;
 206                 }
 207                 t->qid[i] = i;
 208         }
 209         t->nb_qids += num_qids;
 210         if (t->nb_qids > MAX_QIDS)
 211                 return -1;
 212
 213         return 0;
 214 }
 215
 216 static inline int
 217 create_atomic_qids(struct test *t, int num_qids)
 218 {
 219         return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
 220 }
 221
 222 static inline int
 223 create_ordered_qids(struct test *t, int num_qids)
 224 {
 225         return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ORDERED);
 226 }
 227
 228
 229 static inline int
 230 create_unordered_qids(struct test *t, int num_qids)
 231 {
 232         return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_PARALLEL);
 233 }
 234
 235 static inline int
 236 create_directed_qids(struct test *t, int num_qids, const uint8_t ports[])
 237 {
 238         int i;
 239
 240         /* Q creation */
 241         static const struct rte_event_queue_conf conf = {
 242                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 243                         .event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK,
 244         };
 245
 246         for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
 247                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
 248                         printf("%d: error creating qid %d\n", __LINE__, i);
 249                         return -1;
 250                 }
 251                 t->qid[i] = i;
 252
 253                 if (rte_event_port_link(evdev, ports[i - t->nb_qids],
 254                                 &t->qid[i], NULL, 1) != 1) {
 255                         printf("%d: error creating link for qid %d\n",
 256                                         __LINE__, i);
 257                         return -1;
 258                 }
 259         }
 260         t->nb_qids += num_qids;
 261         if (t->nb_qids > MAX_QIDS)
 262                 return -1;
 263
 264         return 0;
 265 }
 266
 267 /* destruction */
 268 static inline int
 269 cleanup(struct test *t __rte_unused)
 270 {
 271         rte_event_dev_stop(evdev);
 272         rte_event_dev_close(evdev);
 273         return 0;
 274 };
 275
 276 struct test_event_dev_stats {
 277         uint64_t rx_pkts;       /**< Total packets received */
 278         uint64_t rx_dropped;    /**< Total packets dropped (Eg Invalid QID) */
 279         uint64_t tx_pkts;       /**< Total packets transmitted */
 280
 281         /** Packets received on this port */
 282         uint64_t port_rx_pkts[MAX_PORTS];
 283         /** Packets dropped on this port */
 284         uint64_t port_rx_dropped[MAX_PORTS];
 285         /** Packets inflight on this port */
 286         uint64_t port_inflight[MAX_PORTS];
 287         /** Packets transmitted on this port */
 288         uint64_t port_tx_pkts[MAX_PORTS];
 289         /** Packets received on this qid */
 290         uint64_t qid_rx_pkts[MAX_QIDS];
 291         /** Packets dropped on this qid */
 292         uint64_t qid_rx_dropped[MAX_QIDS];
 293         /** Packets transmitted on this qid */
 294         uint64_t qid_tx_pkts[MAX_QIDS];
 295 };
 296
 297 static inline int
 298 test_event_dev_stats_get(int dev_id, struct test_event_dev_stats *stats)
 299 {
 300         static uint32_t i;
 301         static uint32_t total_ids[3]; /* rx, tx and drop */
 302         static uint32_t port_rx_pkts_ids[MAX_PORTS];
 303         static uint32_t port_rx_dropped_ids[MAX_PORTS];
 304         static uint32_t port_inflight_ids[MAX_PORTS];
 305         static uint32_t port_tx_pkts_ids[MAX_PORTS];
 306         static uint32_t qid_rx_pkts_ids[MAX_QIDS];
 307         static uint32_t qid_rx_dropped_ids[MAX_QIDS];
 308         static uint32_t qid_tx_pkts_ids[MAX_QIDS];
 309
 310
 311         stats->rx_pkts = rte_event_dev_xstats_by_name_get(dev_id,
 312                         "dev_rx", &total_ids[0]);
 313         stats->rx_dropped = rte_event_dev_xstats_by_name_get(dev_id,
 314                         "dev_drop", &total_ids[1]);
 315         stats->tx_pkts = rte_event_dev_xstats_by_name_get(dev_id,
 316                         "dev_tx", &total_ids[2]);
 317         for (i = 0; i < MAX_PORTS; i++) {
 318                 char name[32];
 319                 snprintf(name, sizeof(name), "port_%u_rx", i);
 320                 stats->port_rx_pkts[i] = rte_event_dev_xstats_by_name_get(
 321                                 dev_id, name, &port_rx_pkts_ids[i]);
 322                 snprintf(name, sizeof(name), "port_%u_drop", i);
 323                 stats->port_rx_dropped[i] = rte_event_dev_xstats_by_name_get(
 324                                 dev_id, name, &port_rx_dropped_ids[i]);
 325                 snprintf(name, sizeof(name), "port_%u_inflight", i);
 326                 stats->port_inflight[i] = rte_event_dev_xstats_by_name_get(
 327                                 dev_id, name, &port_inflight_ids[i]);
 328                 snprintf(name, sizeof(name), "port_%u_tx", i);
 329                 stats->port_tx_pkts[i] = rte_event_dev_xstats_by_name_get(
 330                                 dev_id, name, &port_tx_pkts_ids[i]);
 331         }
 332         for (i = 0; i < MAX_QIDS; i++) {
 333                 char name[32];
 334                 snprintf(name, sizeof(name), "qid_%u_rx", i);
 335                 stats->qid_rx_pkts[i] = rte_event_dev_xstats_by_name_get(
 336                                 dev_id, name, &qid_rx_pkts_ids[i]);
 337                 snprintf(name, sizeof(name), "qid_%u_drop", i);
 338                 stats->qid_rx_dropped[i] = rte_event_dev_xstats_by_name_get(
 339                                 dev_id, name, &qid_rx_dropped_ids[i]);
 340                 snprintf(name, sizeof(name), "qid_%u_tx", i);
 341                 stats->qid_tx_pkts[i] = rte_event_dev_xstats_by_name_get(
 342                                 dev_id, name, &qid_tx_pkts_ids[i]);
 343         }
 344
 345         return 0;
 346 }
 347
 348 /* run_prio_packet_test
 349  * This performs a basic packet priority check on the test instance passed in.
 350  * It is factored out of the main priority tests as the same tests must be
 351  * performed to ensure prioritization of each type of QID.
 352  *
 353  * Requirements:
 354  *  - An initialized test structure, including mempool
 355  *  - t->port[0] is initialized for both Enq / Deq of packets to the QID
 356  *  - t->qid[0] is the QID to be tested
 357  *  - if LB QID, the CQ must be mapped to the QID.
 358  */
 359 static int
 360 run_prio_packet_test(struct test *t)
 361 {
 362         int err;
 363         const uint32_t MAGIC_SEQN[] = {4711, 1234};
 364         const uint32_t PRIORITY[] = {
 365                 RTE_EVENT_DEV_PRIORITY_NORMAL,
 366                 RTE_EVENT_DEV_PRIORITY_HIGHEST
 367         };
 368         unsigned int i;
 369         for (i = 0; i < RTE_DIM(MAGIC_SEQN); i++) {
 370                 /* generate pkt and enqueue */
 371                 struct rte_event ev;
 372                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
 373                 if (!arp) {
 374                         printf("%d: gen of pkt failed\n", __LINE__);
 375                         return -1;
 376                 }
 377                 arp->seqn = MAGIC_SEQN[i];
 378
 379                 ev = (struct rte_event){
 380                         .priority = PRIORITY[i],
 381                         .op = RTE_EVENT_OP_NEW,
 382                         .queue_id = t->qid[0],
 383                         .mbuf = arp
 384                 };
 385                 err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
 386                 if (err < 0) {
 387                         printf("%d: error failed to enqueue\n", __LINE__);
 388                         return -1;
 389                 }
 390         }
 391
 392         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 393
 394         struct test_event_dev_stats stats;
 395         err = test_event_dev_stats_get(evdev, &stats);
 396         if (err) {
 397                 printf("%d: error failed to get stats\n", __LINE__);
 398                 return -1;
 399         }
 400
 401         if (stats.port_rx_pkts[t->port[0]] != 2) {
 402                 printf("%d: error stats incorrect for directed port\n",
 403                                 __LINE__);
 404                 rte_event_dev_dump(evdev, stdout);
 405                 return -1;
 406         }
 407
 408         struct rte_event ev, ev2;
 409         uint32_t deq_pkts;
 410         deq_pkts = rte_event_dequeue_burst(evdev, t->port[0], &ev, 1, 0);
 411         if (deq_pkts != 1) {
 412                 printf("%d: error failed to deq\n", __LINE__);
 413                 rte_event_dev_dump(evdev, stdout);
 414                 return -1;
 415         }
 416         if (ev.mbuf->seqn != MAGIC_SEQN[1]) {
 417                 printf("%d: first packet out not highest priority\n",
 418                                 __LINE__);
 419                 rte_event_dev_dump(evdev, stdout);
 420                 return -1;
 421         }
 422         rte_pktmbuf_free(ev.mbuf);
 423
 424         deq_pkts = rte_event_dequeue_burst(evdev, t->port[0], &ev2, 1, 0);
 425         if (deq_pkts != 1) {
 426                 printf("%d: error failed to deq\n", __LINE__);
 427                 rte_event_dev_dump(evdev, stdout);
 428                 return -1;
 429         }
 430         if (ev2.mbuf->seqn != MAGIC_SEQN[0]) {
 431                 printf("%d: second packet out not lower priority\n",
 432                                 __LINE__);
 433                 rte_event_dev_dump(evdev, stdout);
 434                 return -1;
 435         }
 436         rte_pktmbuf_free(ev2.mbuf);
 437
 438         cleanup(t);
 439         return 0;
 440 }
 441
 442 static int
 443 test_single_directed_packet(struct test *t)
 444 {
 445         const int rx_enq = 0;
 446         const int wrk_enq = 2;
 447         int err;
 448
 449         /* Create instance with 3 directed QIDs going to 3 ports */
 450         if (init(t, 3, 3) < 0 ||
 451                         create_ports(t, 3) < 0 ||
 452                         create_directed_qids(t, 3, t->port) < 0)
 453                 return -1;
 454
 455         if (rte_event_dev_start(evdev) < 0) {
 456                 printf("%d: Error with start call\n", __LINE__);
 457                 return -1;
 458         }
 459
 460         /************** FORWARD ****************/
 461         struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
 462         struct rte_event ev = {
 463                         .op = RTE_EVENT_OP_NEW,
 464                         .queue_id = wrk_enq,
 465                         .mbuf = arp,
 466         };
 467
 468         if (!arp) {
 469                 printf("%d: gen of pkt failed\n", __LINE__);
 470                 return -1;
 471         }
 472
 473         const uint32_t MAGIC_SEQN = 4711;
 474         arp->seqn = MAGIC_SEQN;
 475
 476         /* generate pkt and enqueue */
 477         err = rte_event_enqueue_burst(evdev, rx_enq, &ev, 1);
 478         if (err < 0) {
 479                 printf("%d: error failed to enqueue\n", __LINE__);
 480                 return -1;
 481         }
 482
 483         /* Run schedule() as dir packets may need to be re-ordered */
 484         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 485
 486         struct test_event_dev_stats stats;
 487         err = test_event_dev_stats_get(evdev, &stats);
 488         if (err) {
 489                 printf("%d: error failed to get stats\n", __LINE__);
 490                 return -1;
 491         }
 492
 493         if (stats.port_rx_pkts[rx_enq] != 1) {
 494                 printf("%d: error stats incorrect for directed port\n",
 495                                 __LINE__);
 496                 return -1;
 497         }
 498
 499         uint32_t deq_pkts;
 500         deq_pkts = rte_event_dequeue_burst(evdev, wrk_enq, &ev, 1, 0);
 501         if (deq_pkts != 1) {
 502                 printf("%d: error failed to deq\n", __LINE__);
 503                 return -1;
 504         }
 505
 506         err = test_event_dev_stats_get(evdev, &stats);
 507         if (stats.port_rx_pkts[wrk_enq] != 0 &&
 508                         stats.port_rx_pkts[wrk_enq] != 1) {
 509                 printf("%d: error directed stats post-dequeue\n", __LINE__);
 510                 return -1;
 511         }
 512
 513         if (ev.mbuf->seqn != MAGIC_SEQN) {
 514                 printf("%d: error magic sequence number not dequeued\n",
 515                                 __LINE__);
 516                 return -1;
 517         }
 518
 519         rte_pktmbuf_free(ev.mbuf);
 520         cleanup(t);
 521         return 0;
 522 }
 523
 524 static int
 525 test_directed_forward_credits(struct test *t)
 526 {
 527         uint32_t i;
 528         int32_t err;
 529
 530         if (init(t, 1, 1) < 0 ||
 531                         create_ports(t, 1) < 0 ||
 532                         create_directed_qids(t, 1, t->port) < 0)
 533                 return -1;
 534
 535         if (rte_event_dev_start(evdev) < 0) {
 536                 printf("%d: Error with start call\n", __LINE__);
 537                 return -1;
 538         }
 539
 540         struct rte_event ev = {
 541                         .op = RTE_EVENT_OP_NEW,
 542                         .queue_id = 0,
 543         };
 544
 545         for (i = 0; i < 1000; i++) {
 546                 err = rte_event_enqueue_burst(evdev, 0, &ev, 1);
 547                 if (err < 0) {
 548                         printf("%d: error failed to enqueue\n", __LINE__);
 549                         return -1;
 550                 }
 551                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
 552
 553                 uint32_t deq_pkts;
 554                 deq_pkts = rte_event_dequeue_burst(evdev, 0, &ev, 1, 0);
 555                 if (deq_pkts != 1) {
 556                         printf("%d: error failed to deq\n", __LINE__);
 557                         return -1;
 558                 }
 559
 560                 /* re-write event to be a forward, and continue looping it */
 561                 ev.op = RTE_EVENT_OP_FORWARD;
 562         }
 563
 564         cleanup(t);
 565         return 0;
 566 }
 567
 568
 569 static int
 570 test_priority_directed(struct test *t)
 571 {
 572         if (init(t, 1, 1) < 0 ||
 573                         create_ports(t, 1) < 0 ||
 574                         create_directed_qids(t, 1, t->port) < 0) {
 575                 printf("%d: Error initializing device\n", __LINE__);
 576                 return -1;
 577         }
 578
 579         if (rte_event_dev_start(evdev) < 0) {
 580                 printf("%d: Error with start call\n", __LINE__);
 581                 return -1;
 582         }
 583
 584         return run_prio_packet_test(t);
 585 }
 586
 587 static int
 588 test_priority_atomic(struct test *t)
 589 {
 590         if (init(t, 1, 1) < 0 ||
 591                         create_ports(t, 1) < 0 ||
 592                         create_atomic_qids(t, 1) < 0) {
 593                 printf("%d: Error initializing device\n", __LINE__);
 594                 return -1;
 595         }
 596
 597         /* map the QID */
 598         if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) {
 599                 printf("%d: error mapping qid to port\n", __LINE__);
 600                 return -1;
 601         }
 602         if (rte_event_dev_start(evdev) < 0) {
 603                 printf("%d: Error with start call\n", __LINE__);
 604                 return -1;
 605         }
 606
 607         return run_prio_packet_test(t);
 608 }
 609
 610 static int
 611 test_priority_ordered(struct test *t)
 612 {
 613         if (init(t, 1, 1) < 0 ||
 614                         create_ports(t, 1) < 0 ||
 615                         create_ordered_qids(t, 1) < 0) {
 616                 printf("%d: Error initializing device\n", __LINE__);
 617                 return -1;
 618         }
 619
 620         /* map the QID */
 621         if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) {
 622                 printf("%d: error mapping qid to port\n", __LINE__);
 623                 return -1;
 624         }
 625         if (rte_event_dev_start(evdev) < 0) {
 626                 printf("%d: Error with start call\n", __LINE__);
 627                 return -1;
 628         }
 629
 630         return run_prio_packet_test(t);
 631 }
 632
 633 static int
 634 test_priority_unordered(struct test *t)
 635 {
 636         if (init(t, 1, 1) < 0 ||
 637                         create_ports(t, 1) < 0 ||
 638                         create_unordered_qids(t, 1) < 0) {
 639                 printf("%d: Error initializing device\n", __LINE__);
 640                 return -1;
 641         }
 642
 643         /* map the QID */
 644         if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) {
 645                 printf("%d: error mapping qid to port\n", __LINE__);
 646                 return -1;
 647         }
 648         if (rte_event_dev_start(evdev) < 0) {
 649                 printf("%d: Error with start call\n", __LINE__);
 650                 return -1;
 651         }
 652
 653         return run_prio_packet_test(t);
 654 }
 655
 656 static int
 657 burst_packets(struct test *t)
 658 {
 659         /************** CONFIG ****************/
 660         uint32_t i;
 661         int err;
 662         int ret;
 663
 664         /* Create instance with 2 ports and 2 queues */
 665         if (init(t, 2, 2) < 0 ||
 666                         create_ports(t, 2) < 0 ||
 667                         create_atomic_qids(t, 2) < 0) {
 668                 printf("%d: Error initializing device\n", __LINE__);
 669                 return -1;
 670         }
 671
 672         /* CQ mapping to QID */
 673         ret = rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1);
 674         if (ret != 1) {
 675                 printf("%d: error mapping lb qid0\n", __LINE__);
 676                 return -1;
 677         }
 678         ret = rte_event_port_link(evdev, t->port[1], &t->qid[1], NULL, 1);
 679         if (ret != 1) {
 680                 printf("%d: error mapping lb qid1\n", __LINE__);
 681                 return -1;
 682         }
 683
 684         if (rte_event_dev_start(evdev) < 0) {
 685                 printf("%d: Error with start call\n", __LINE__);
 686                 return -1;
 687         }
 688
 689         /************** FORWARD ****************/
 690         const uint32_t rx_port = 0;
 691         const uint32_t NUM_PKTS = 2;
 692
 693         for (i = 0; i < NUM_PKTS; i++) {
 694                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
 695                 if (!arp) {
 696                         printf("%d: error generating pkt\n", __LINE__);
 697                         return -1;
 698                 }
 699
 700                 struct rte_event ev = {
 701                                 .op = RTE_EVENT_OP_NEW,
 702                                 .queue_id = i % 2,
 703                                 .flow_id = i % 3,
 704                                 .mbuf = arp,
 705                 };
 706                 /* generate pkt and enqueue */
 707                 err = rte_event_enqueue_burst(evdev, t->port[rx_port], &ev, 1);
 708                 if (err < 0) {
 709                         printf("%d: Failed to enqueue\n", __LINE__);
 710                         return -1;
 711                 }
 712         }
 713         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 714
 715         /* Check stats for all NUM_PKTS arrived to sched core */
 716         struct test_event_dev_stats stats;
 717
 718         err = test_event_dev_stats_get(evdev, &stats);
 719         if (err) {
 720                 printf("%d: failed to get stats\n", __LINE__);
 721                 return -1;
 722         }
 723         if (stats.rx_pkts != NUM_PKTS || stats.tx_pkts != NUM_PKTS) {
 724                 printf("%d: Sched core didn't receive all %d pkts\n",
 725                                 __LINE__, NUM_PKTS);
 726                 rte_event_dev_dump(evdev, stdout);
 727                 return -1;
 728         }
 729
 730         uint32_t deq_pkts;
 731         int p;
 732
 733         deq_pkts = 0;
 734         /******** DEQ QID 1 *******/
 735         do {
 736                 struct rte_event ev;
 737                 p = rte_event_dequeue_burst(evdev, t->port[0], &ev, 1, 0);
 738                 deq_pkts += p;
 739                 rte_pktmbuf_free(ev.mbuf);
 740         } while (p);
 741
 742         if (deq_pkts != NUM_PKTS/2) {
 743                 printf("%d: Half of NUM_PKTS didn't arrive at port 1\n",
 744                                 __LINE__);
 745                 return -1;
 746         }
 747
 748         /******** DEQ QID 2 *******/
 749         deq_pkts = 0;
 750         do {
 751                 struct rte_event ev;
 752                 p = rte_event_dequeue_burst(evdev, t->port[1], &ev, 1, 0);
 753                 deq_pkts += p;
 754                 rte_pktmbuf_free(ev.mbuf);
 755         } while (p);
 756         if (deq_pkts != NUM_PKTS/2) {
 757                 printf("%d: Half of NUM_PKTS didn't arrive at port 2\n",
 758                                 __LINE__);
 759                 return -1;
 760         }
 761
 762         cleanup(t);
 763         return 0;
 764 }
 765
 766 static int
 767 abuse_inflights(struct test *t)
 768 {
 769         const int rx_enq = 0;
 770         const int wrk_enq = 2;
 771         int err;
 772
 773         /* Create instance with 4 ports */
 774         if (init(t, 1, 4) < 0 ||
 775                         create_ports(t, 4) < 0 ||
 776                         create_atomic_qids(t, 1) < 0) {
 777                 printf("%d: Error initializing device\n", __LINE__);
 778                 return -1;
 779         }
 780
 781         /* CQ mapping to QID */
 782         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
 783         if (err != 1) {
 784                 printf("%d: error mapping lb qid\n", __LINE__);
 785                 cleanup(t);
 786                 return -1;
 787         }
 788
 789         if (rte_event_dev_start(evdev) < 0) {
 790                 printf("%d: Error with start call\n", __LINE__);
 791                 return -1;
 792         }
 793
 794         /* Enqueue op only */
 795         err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &release_ev, 1);
 796         if (err < 0) {
 797                 printf("%d: Failed to enqueue\n", __LINE__);
 798                 return -1;
 799         }
 800
 801         /* schedule */
 802         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 803
 804         struct test_event_dev_stats stats;
 805
 806         err = test_event_dev_stats_get(evdev, &stats);
 807         if (err) {
 808                 printf("%d: failed to get stats\n", __LINE__);
 809                 return -1;
 810         }
 811
 812         if (stats.rx_pkts != 0 ||
 813                         stats.tx_pkts != 0 ||
 814                         stats.port_inflight[wrk_enq] != 0) {
 815                 printf("%d: Sched core didn't handle pkt as expected\n",
 816                                 __LINE__);
 817                 return -1;
 818         }
 819
 820         cleanup(t);
 821         return 0;
 822 }
 823
 824 static int
 825 xstats_tests(struct test *t)
 826 {
 827         const int wrk_enq = 2;
 828         int err;
 829
 830         /* Create instance with 4 ports */
 831         if (init(t, 1, 4) < 0 ||
 832                         create_ports(t, 4) < 0 ||
 833                         create_atomic_qids(t, 1) < 0) {
 834                 printf("%d: Error initializing device\n", __LINE__);
 835                 return -1;
 836         }
 837
 838         /* CQ mapping to QID */
 839         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
 840         if (err != 1) {
 841                 printf("%d: error mapping lb qid\n", __LINE__);
 842                 cleanup(t);
 843                 return -1;
 844         }
 845
 846         if (rte_event_dev_start(evdev) < 0) {
 847                 printf("%d: Error with start call\n", __LINE__);
 848                 return -1;
 849         }
 850
 851         const uint32_t XSTATS_MAX = 1024;
 852
 853         uint32_t i;
 854         uint32_t ids[XSTATS_MAX];
 855         uint64_t values[XSTATS_MAX];
 856         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
 857
 858         for (i = 0; i < XSTATS_MAX; i++)
 859                 ids[i] = i;
 860
 861         /* Device names / values */
 862         int ret = rte_event_dev_xstats_names_get(evdev,
 863                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 864                                         0, xstats_names, ids, XSTATS_MAX);
 865         if (ret != 6) {
 866                 printf("%d: expected 6 stats, got return %d\n", __LINE__, ret);
 867                 return -1;
 868         }
 869         ret = rte_event_dev_xstats_get(evdev,
 870                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 871                                         0, ids, values, ret);
 872         if (ret != 6) {
 873                 printf("%d: expected 6 stats, got return %d\n", __LINE__, ret);
 874                 return -1;
 875         }
 876
 877         /* Port names / values */
 878         ret = rte_event_dev_xstats_names_get(evdev,
 879                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
 880                                         xstats_names, ids, XSTATS_MAX);
 881         if (ret != 21) {
 882                 printf("%d: expected 21 stats, got return %d\n", __LINE__, ret);
 883                 return -1;
 884         }
 885         ret = rte_event_dev_xstats_get(evdev,
 886                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
 887                                         ids, values, ret);
 888         if (ret != 21) {
 889                 printf("%d: expected 21 stats, got return %d\n", __LINE__, ret);
 890                 return -1;
 891         }
 892
 893         /* Queue names / values */
 894         ret = rte_event_dev_xstats_names_get(evdev,
 895                                         RTE_EVENT_DEV_XSTATS_QUEUE,
 896                                         0, xstats_names, ids, XSTATS_MAX);
 897         if (ret != 17) {
 898                 printf("%d: expected 17 stats, got return %d\n", __LINE__, ret);
 899                 return -1;
 900         }
 901
 902         /* NEGATIVE TEST: with wrong queue passed, 0 stats should be returned */
 903         ret = rte_event_dev_xstats_get(evdev,
 904                                         RTE_EVENT_DEV_XSTATS_QUEUE,
 905                                         1, ids, values, ret);
 906         if (ret != -EINVAL) {
 907                 printf("%d: expected 0 stats, got return %d\n", __LINE__, ret);
 908                 return -1;
 909         }
 910
 911         ret = rte_event_dev_xstats_get(evdev,
 912                                         RTE_EVENT_DEV_XSTATS_QUEUE,
 913                                         0, ids, values, ret);
 914         if (ret != 17) {
 915                 printf("%d: expected 17 stats, got return %d\n", __LINE__, ret);
 916                 return -1;
 917         }
 918
 919         /* enqueue packets to check values */
 920         for (i = 0; i < 3; i++) {
 921                 struct rte_event ev;
 922                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
 923                 if (!arp) {
 924                         printf("%d: gen of pkt failed\n", __LINE__);
 925                         return -1;
 926                 }
 927                 ev.queue_id = t->qid[i];
 928                 ev.op = RTE_EVENT_OP_NEW;
 929                 ev.mbuf = arp;
 930                 ev.flow_id = 7;
 931                 arp->seqn = i;
 932
 933                 int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
 934                 if (err != 1) {
 935                         printf("%d: Failed to enqueue\n", __LINE__);
 936                         return -1;
 937                 }
 938         }
 939
 940         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 941
 942         /* Device names / values */
 943         int num_stats = rte_event_dev_xstats_names_get(evdev,
 944                                         RTE_EVENT_DEV_XSTATS_DEVICE, 0,
 945                                         xstats_names, ids, XSTATS_MAX);
 946         if (num_stats < 0)
 947                 goto fail;
 948         ret = rte_event_dev_xstats_get(evdev,
 949                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 950                                         0, ids, values, num_stats);
 951         static const uint64_t expected[] = {3, 3, 0, 1, 0, 0};
 952         for (i = 0; (signed int)i < ret; i++) {
 953                 if (expected[i] != values[i]) {
 954                         printf(
 955                                 "%d Error xstat %d (id %d) %s : %"PRIu64
 956                                 ", expect %"PRIu64"\n",
 957                                 __LINE__, i, ids[i], xstats_names[i].name,
 958                                 values[i], expected[i]);
 959                         goto fail;
 960                 }
 961         }
 962
 963         ret = rte_event_dev_xstats_reset(evdev, RTE_EVENT_DEV_XSTATS_DEVICE,
 964                                         0, NULL, 0);
 965
 966         /* ensure reset statistics are zero-ed */
 967         static const uint64_t expected_zero[] = {0, 0, 0, 0, 0, 0};
 968         ret = rte_event_dev_xstats_get(evdev,
 969                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 970                                         0, ids, values, num_stats);
 971         for (i = 0; (signed int)i < ret; i++) {
 972                 if (expected_zero[i] != values[i]) {
 973                         printf(
 974                                 "%d Error, xstat %d (id %d) %s : %"PRIu64
 975                                 ", expect %"PRIu64"\n",
 976                                 __LINE__, i, ids[i], xstats_names[i].name,
 977                                 values[i], expected_zero[i]);
 978                         goto fail;
 979                 }
 980         }
 981
 982         /* port reset checks */
 983         num_stats = rte_event_dev_xstats_names_get(evdev,
 984                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
 985                                         xstats_names, ids, XSTATS_MAX);
 986         if (num_stats < 0)
 987                 goto fail;
 988         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_PORT,
 989                                         0, ids, values, num_stats);
 990
 991         static const uint64_t port_expected[] = {
 992                 3 /* rx */,
 993                 0 /* tx */,
 994                 0 /* drop */,
 995                 0 /* inflights */,
 996                 0 /* avg pkt cycles */,
 997                 29 /* credits */,
 998                 0 /* rx ring used */,
 999                 4096 /* rx ring free */,
1000                 0 /* cq ring used */,
1001                 32 /* cq ring free */,
1002                 0 /* dequeue calls */,
1003                 /* 10 dequeue burst buckets */
1004                 0, 0, 0, 0, 0,
1005                 0, 0, 0, 0, 0,
1006         };
1007         if (ret != RTE_DIM(port_expected)) {
1008                 printf(
1009                         "%s %d: wrong number of port stats (%d), expected %zu\n",
1010                         __func__, __LINE__, ret, RTE_DIM(port_expected));
1011         }
1012
1013         for (i = 0; (signed int)i < ret; i++) {
1014                 if (port_expected[i] != values[i]) {
1015                         printf(
1016                                 "%s : %d: Error stat %s is %"PRIu64
1017                                 ", expected %"PRIu64"\n",
1018                                 __func__, __LINE__, xstats_names[i].name,
1019                                 values[i], port_expected[i]);
1020                         goto fail;
1021                 }
1022         }
1023
1024         ret = rte_event_dev_xstats_reset(evdev, RTE_EVENT_DEV_XSTATS_PORT,
1025                                         0, NULL, 0);
1026
1027         /* ensure reset statistics are zero-ed */
1028         static const uint64_t port_expected_zero[] = {
1029                 0 /* rx */,
1030                 0 /* tx */,
1031                 0 /* drop */,
1032                 0 /* inflights */,
1033                 0 /* avg pkt cycles */,
1034                 29 /* credits */,
1035                 0 /* rx ring used */,
1036                 4096 /* rx ring free */,
1037                 0 /* cq ring used */,
1038                 32 /* cq ring free */,
1039                 0 /* dequeue calls */,
1040                 /* 10 dequeue burst buckets */
1041                 0, 0, 0, 0, 0,
1042                 0, 0, 0, 0, 0,
1043         };
1044         ret = rte_event_dev_xstats_get(evdev,
1045                                         RTE_EVENT_DEV_XSTATS_PORT,
1046                                         0, ids, values, num_stats);
1047         for (i = 0; (signed int)i < ret; i++) {
1048                 if (port_expected_zero[i] != values[i]) {
1049                         printf(
1050                                 "%d, Error, xstat %d (id %d) %s : %"PRIu64
1051                                 ", expect %"PRIu64"\n",
1052                                 __LINE__, i, ids[i], xstats_names[i].name,
1053                                 values[i], port_expected_zero[i]);
1054                         goto fail;
1055                 }
1056         }
1057
1058         /* QUEUE STATS TESTS */
1059         num_stats = rte_event_dev_xstats_names_get(evdev,
1060                                                 RTE_EVENT_DEV_XSTATS_QUEUE, 0,
1061                                                 xstats_names, ids, XSTATS_MAX);
1062         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE,
1063                                         0, ids, values, num_stats);
1064         if (ret < 0) {
1065                 printf("xstats get returned %d\n", ret);
1066                 goto fail;
1067         }
1068         if ((unsigned int)ret > XSTATS_MAX)
1069                 printf("%s %d: more xstats available than space\n",
1070                                 __func__, __LINE__);
1071
1072         static const uint64_t queue_expected[] = {
1073                 3 /* rx */,
1074                 3 /* tx */,
1075                 0 /* drop */,
1076                 3 /* inflights */,
1077                 512 /* iq size */,
1078                 0, 0, 0, 0, /* iq 0, 1, 2, 3 used */
1079                 /* QID-to-Port: pinned_flows, packets */
1080                 0, 0,
1081                 0, 0,
1082                 1, 3,
1083                 0, 0,
1084         };
1085         for (i = 0; (signed int)i < ret; i++) {
1086                 if (queue_expected[i] != values[i]) {
1087                         printf(
1088                                 "%d, Error, xstat %d (id %d) %s : %"PRIu64
1089                                 ", expect %"PRIu64"\n",
1090                                 __LINE__, i, ids[i], xstats_names[i].name,
1091                                 values[i], queue_expected[i]);
1092                         goto fail;
1093                 }
1094         }
1095
1096         /* Reset the queue stats here */
1097         ret = rte_event_dev_xstats_reset(evdev,
1098                                         RTE_EVENT_DEV_XSTATS_QUEUE, 0,
1099                                         NULL,
1100                                         0);
1101
1102         /* Verify that the resetable stats are reset, and others are not */
1103         static const uint64_t queue_expected_zero[] = {
1104                 0 /* rx */,
1105                 0 /* tx */,
1106                 0 /* drop */,
1107                 3 /* inflight */,
1108                 512 /* iq size */,
1109                 0, 0, 0, 0, /* 4 iq used */
1110                 /* QID-to-Port: pinned_flows, packets */
1111                 0, 0,
1112                 0, 0,
1113                 1, 0,
1114                 0, 0,
1115         };
1116
1117         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE, 0,
1118                                         ids, values, num_stats);
1119         int fails = 0;
1120         for (i = 0; (signed int)i < ret; i++) {
1121                 if (queue_expected_zero[i] != values[i]) {
1122                         printf(
1123                                 "%d, Error, xstat %d (id %d) %s : %"PRIu64
1124                                 ", expect %"PRIu64"\n",
1125                                 __LINE__, i, ids[i], xstats_names[i].name,
1126                                 values[i], queue_expected_zero[i]);
1127                         fails++;
1128                 }
1129         }
1130         if (fails) {
1131                 printf("%d : %d of values were not as expected above\n",
1132                                 __LINE__, fails);
1133                 goto fail;
1134         }
1135
1136         cleanup(t);
1137         return 0;
1138
1139 fail:
1140         rte_event_dev_dump(0, stdout);
1141         cleanup(t);
1142         return -1;
1143 }
1144
1145
1146 static int
1147 xstats_id_abuse_tests(struct test *t)
1148 {
1149         int err;
1150         const uint32_t XSTATS_MAX = 1024;
1151         const uint32_t link_port = 2;
1152
1153         uint32_t ids[XSTATS_MAX];
1154         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
1155
1156         /* Create instance with 4 ports */
1157         if (init(t, 1, 4) < 0 ||
1158                         create_ports(t, 4) < 0 ||
1159                         create_atomic_qids(t, 1) < 0) {
1160                 printf("%d: Error initializing device\n", __LINE__);
1161                 goto fail;
1162         }
1163
1164         err = rte_event_port_link(evdev, t->port[link_port], NULL, NULL, 0);
1165         if (err != 1) {
1166                 printf("%d: error mapping lb qid\n", __LINE__);
1167                 goto fail;
1168         }
1169
1170         if (rte_event_dev_start(evdev) < 0) {
1171                 printf("%d: Error with start call\n", __LINE__);
1172                 goto fail;
1173         }
1174
1175         /* no test for device, as it ignores the port/q number */
1176         int num_stats = rte_event_dev_xstats_names_get(evdev,
1177                                         RTE_EVENT_DEV_XSTATS_PORT,
1178                                         UINT8_MAX-1, xstats_names, ids,
1179                                         XSTATS_MAX);
1180         if (num_stats != 0) {
1181                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1182                                 0, num_stats);
1183                 goto fail;
1184         }
1185
1186         num_stats = rte_event_dev_xstats_names_get(evdev,
1187                                         RTE_EVENT_DEV_XSTATS_QUEUE,
1188                                         UINT8_MAX-1, xstats_names, ids,
1189                                         XSTATS_MAX);
1190         if (num_stats != 0) {
1191                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1192                                 0, num_stats);
1193                 goto fail;
1194         }
1195
1196         cleanup(t);
1197         return 0;
1198 fail:
1199         cleanup(t);
1200         return -1;
1201 }
1202
1203 static int
1204 port_reconfig_credits(struct test *t)
1205 {
1206         if (init(t, 1, 1) < 0) {
1207                 printf("%d: Error initializing device\n", __LINE__);
1208                 return -1;
1209         }
1210
1211         uint32_t i;
1212         const uint32_t NUM_ITERS = 32;
1213         for (i = 0; i < NUM_ITERS; i++) {
1214                 const struct rte_event_queue_conf conf = {
1215                         .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1216                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1217                         .nb_atomic_flows = 1024,
1218                         .nb_atomic_order_sequences = 1024,
1219                 };
1220                 if (rte_event_queue_setup(evdev, 0, &conf) < 0) {
1221                         printf("%d: error creating qid\n", __LINE__);
1222                         return -1;
1223                 }
1224                 t->qid[0] = 0;
1225
1226                 static const struct rte_event_port_conf port_conf = {
1227                                 .new_event_threshold = 128,
1228                                 .dequeue_depth = 32,
1229                                 .enqueue_depth = 64,
1230                 };
1231                 if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
1232                         printf("%d Error setting up port\n", __LINE__);
1233                         return -1;
1234                 }
1235
1236                 int links = rte_event_port_link(evdev, 0, NULL, NULL, 0);
1237                 if (links != 1) {
1238                         printf("%d: error mapping lb qid\n", __LINE__);
1239                         goto fail;
1240                 }
1241
1242                 if (rte_event_dev_start(evdev) < 0) {
1243                         printf("%d: Error with start call\n", __LINE__);
1244                         goto fail;
1245                 }
1246
1247                 const uint32_t NPKTS = 1;
1248                 uint32_t j;
1249                 for (j = 0; j < NPKTS; j++) {
1250                         struct rte_event ev;
1251                         struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1252                         if (!arp) {
1253                                 printf("%d: gen of pkt failed\n", __LINE__);
1254                                 goto fail;
1255                         }
1256                         ev.queue_id = t->qid[0];
1257                         ev.op = RTE_EVENT_OP_NEW;
1258                         ev.mbuf = arp;
1259                         int err = rte_event_enqueue_burst(evdev, 0, &ev, 1);
1260                         if (err != 1) {
1261                                 printf("%d: Failed to enqueue\n", __LINE__);
1262                                 rte_event_dev_dump(0, stdout);
1263                                 goto fail;
1264                         }
1265                 }
1266
1267                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
1268
1269                 struct rte_event ev[NPKTS];
1270                 int deq = rte_event_dequeue_burst(evdev, t->port[0], ev,
1271                                                         NPKTS, 0);
1272                 if (deq != 1)
1273                         printf("%d error; no packet dequeued\n", __LINE__);
1274
1275                 /* let cleanup below stop the device on last iter */
1276                 if (i != NUM_ITERS-1)
1277                         rte_event_dev_stop(evdev);
1278         }
1279
1280         cleanup(t);
1281         return 0;
1282 fail:
1283         cleanup(t);
1284         return -1;
1285 }
1286
1287 static int
1288 port_single_lb_reconfig(struct test *t)
1289 {
1290         if (init(t, 2, 2) < 0) {
1291                 printf("%d: Error initializing device\n", __LINE__);
1292                 goto fail;
1293         }
1294
1295         static const struct rte_event_queue_conf conf_lb_atomic = {
1296                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1297                 .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1298                 .nb_atomic_flows = 1024,
1299                 .nb_atomic_order_sequences = 1024,
1300         };
1301         if (rte_event_queue_setup(evdev, 0, &conf_lb_atomic) < 0) {
1302                 printf("%d: error creating qid\n", __LINE__);
1303                 goto fail;
1304         }
1305
1306         static const struct rte_event_queue_conf conf_single_link = {
1307                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1308                 .event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK,
1309         };
1310         if (rte_event_queue_setup(evdev, 1, &conf_single_link) < 0) {
1311                 printf("%d: error creating qid\n", __LINE__);
1312                 goto fail;
1313         }
1314
1315         struct rte_event_port_conf port_conf = {
1316                 .new_event_threshold = 128,
1317                 .dequeue_depth = 32,
1318                 .enqueue_depth = 64,
1319         };
1320         if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
1321                 printf("%d Error setting up port\n", __LINE__);
1322                 goto fail;
1323         }
1324         if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
1325                 printf("%d Error setting up port\n", __LINE__);
1326                 goto fail;
1327         }
1328
1329         /* link port to lb queue */
1330         uint8_t queue_id = 0;
1331         if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
1332                 printf("%d: error creating link for qid\n", __LINE__);
1333                 goto fail;
1334         }
1335
1336         int ret = rte_event_port_unlink(evdev, 0, &queue_id, 1);
1337         if (ret != 1) {
1338                 printf("%d: Error unlinking lb port\n", __LINE__);
1339                 goto fail;
1340         }
1341
1342         queue_id = 1;
1343         if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
1344                 printf("%d: error creating link for qid\n", __LINE__);
1345                 goto fail;
1346         }
1347
1348         queue_id = 0;
1349         int err = rte_event_port_link(evdev, 1, &queue_id, NULL, 1);
1350         if (err != 1) {
1351                 printf("%d: error mapping lb qid\n", __LINE__);
1352                 goto fail;
1353         }
1354
1355         if (rte_event_dev_start(evdev) < 0) {
1356                 printf("%d: Error with start call\n", __LINE__);
1357                 goto fail;
1358         }
1359
1360         cleanup(t);
1361         return 0;
1362 fail:
1363         cleanup(t);
1364         return -1;
1365 }
1366
1367 static int
1368 xstats_brute_force(struct test *t)
1369 {
1370         uint32_t i;
1371         const uint32_t XSTATS_MAX = 1024;
1372         uint32_t ids[XSTATS_MAX];
1373         uint64_t values[XSTATS_MAX];
1374         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
1375
1376
1377         /* Create instance with 4 ports */
1378         if (init(t, 1, 4) < 0 ||
1379                         create_ports(t, 4) < 0 ||
1380                         create_atomic_qids(t, 1) < 0) {
1381                 printf("%d: Error initializing device\n", __LINE__);
1382                 return -1;
1383         }
1384
1385         int err = rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1386         if (err != 1) {
1387                 printf("%d: error mapping lb qid\n", __LINE__);
1388                 goto fail;
1389         }
1390
1391         if (rte_event_dev_start(evdev) < 0) {
1392                 printf("%d: Error with start call\n", __LINE__);
1393                 goto fail;
1394         }
1395
1396         for (i = 0; i < XSTATS_MAX; i++)
1397                 ids[i] = i;
1398
1399         for (i = 0; i < 3; i++) {
1400                 uint32_t mode = RTE_EVENT_DEV_XSTATS_DEVICE + i;
1401                 uint32_t j;
1402                 for (j = 0; j < UINT8_MAX; j++) {
1403                         rte_event_dev_xstats_names_get(evdev, mode,
1404                                 j, xstats_names, ids, XSTATS_MAX);
1405
1406                         rte_event_dev_xstats_get(evdev, mode, j, ids,
1407                                                  values, XSTATS_MAX);
1408                 }
1409         }
1410
1411         cleanup(t);
1412         return 0;
1413 fail:
1414         cleanup(t);
1415         return -1;
1416 }
1417
1418 static int
1419 xstats_id_reset_tests(struct test *t)
1420 {
1421         const int wrk_enq = 2;
1422         int err;
1423
1424         /* Create instance with 4 ports */
1425         if (init(t, 1, 4) < 0 ||
1426                         create_ports(t, 4) < 0 ||
1427                         create_atomic_qids(t, 1) < 0) {
1428                 printf("%d: Error initializing device\n", __LINE__);
1429                 return -1;
1430         }
1431
1432         /* CQ mapping to QID */
1433         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
1434         if (err != 1) {
1435                 printf("%d: error mapping lb qid\n", __LINE__);
1436                 goto fail;
1437         }
1438
1439         if (rte_event_dev_start(evdev) < 0) {
1440                 printf("%d: Error with start call\n", __LINE__);
1441                 goto fail;
1442         }
1443
1444 #define XSTATS_MAX 1024
1445         int ret;
1446         uint32_t i;
1447         uint32_t ids[XSTATS_MAX];
1448         uint64_t values[XSTATS_MAX];
1449         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
1450
1451         for (i = 0; i < XSTATS_MAX; i++)
1452                 ids[i] = i;
1453
1454 #define NUM_DEV_STATS 6
1455         /* Device names / values */
1456         int num_stats = rte_event_dev_xstats_names_get(evdev,
1457                                         RTE_EVENT_DEV_XSTATS_DEVICE,
1458                                         0, xstats_names, ids, XSTATS_MAX);
1459         if (num_stats != NUM_DEV_STATS) {
1460                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1461                                 NUM_DEV_STATS, num_stats);
1462                 goto fail;
1463         }
1464         ret = rte_event_dev_xstats_get(evdev,
1465                                         RTE_EVENT_DEV_XSTATS_DEVICE,
1466                                         0, ids, values, num_stats);
1467         if (ret != NUM_DEV_STATS) {
1468                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1469                                 NUM_DEV_STATS, ret);
1470                 goto fail;
1471         }
1472
1473 #define NPKTS 7
1474         for (i = 0; i < NPKTS; i++) {
1475                 struct rte_event ev;
1476                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1477                 if (!arp) {
1478                         printf("%d: gen of pkt failed\n", __LINE__);
1479                         goto fail;
1480                 }
1481                 ev.queue_id = t->qid[i];
1482                 ev.op = RTE_EVENT_OP_NEW;
1483                 ev.mbuf = arp;
1484                 arp->seqn = i;
1485
1486                 int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
1487                 if (err != 1) {
1488                         printf("%d: Failed to enqueue\n", __LINE__);
1489                         goto fail;
1490                 }
1491         }
1492
1493         rte_service_run_iter_on_app_lcore(t->service_id, 1);
1494
1495         static const char * const dev_names[] = {
1496                 "dev_rx", "dev_tx", "dev_drop", "dev_sched_calls",
1497                 "dev_sched_no_iq_enq", "dev_sched_no_cq_enq",
1498         };
1499         uint64_t dev_expected[] = {NPKTS, NPKTS, 0, 1, 0, 0};
1500         for (i = 0; (int)i < ret; i++) {
1501                 unsigned int id;
1502                 uint64_t val = rte_event_dev_xstats_by_name_get(evdev,
1503                                                                 dev_names[i],
1504                                                                 &id);
1505                 if (id != i) {
1506                         printf("%d: %s id incorrect, expected %d got %d\n",
1507                                         __LINE__, dev_names[i], i, id);
1508                         goto fail;
1509                 }
1510                 if (val != dev_expected[i]) {
1511                         printf("%d: %s value incorrect, expected %"
1512                                 PRIu64" got %d\n", __LINE__, dev_names[i],
1513                                 dev_expected[i], id);
1514                         goto fail;
1515                 }
1516                 /* reset to zero */
1517                 int reset_ret = rte_event_dev_xstats_reset(evdev,
1518                                                 RTE_EVENT_DEV_XSTATS_DEVICE, 0,
1519                                                 &id,
1520                                                 1);
1521                 if (reset_ret) {
1522                         printf("%d: failed to reset successfully\n", __LINE__);
1523                         goto fail;
1524                 }
1525                 dev_expected[i] = 0;
1526                 /* check value again */
1527                 val = rte_event_dev_xstats_by_name_get(evdev, dev_names[i], 0);
1528                 if (val != dev_expected[i]) {
1529                         printf("%d: %s value incorrect, expected %"PRIu64
1530                                 " got %"PRIu64"\n", __LINE__, dev_names[i],
1531                                 dev_expected[i], val);
1532                         goto fail;
1533                 }
1534         };
1535
1536 /* 48 is stat offset from start of the devices whole xstats.
1537  * This WILL break every time we add a statistic to a port
1538  * or the device, but there is no other way to test
1539  */
1540 #define PORT_OFF 48
1541 /* num stats for the tested port. CQ size adds more stats to a port */
1542 #define NUM_PORT_STATS 21
1543 /* the port to test. */
1544 #define PORT 2
1545         num_stats = rte_event_dev_xstats_names_get(evdev,
1546                                         RTE_EVENT_DEV_XSTATS_PORT, PORT,
1547                                         xstats_names, ids, XSTATS_MAX);
1548         if (num_stats != NUM_PORT_STATS) {
1549                 printf("%d: expected %d stats, got return %d\n",
1550                         __LINE__, NUM_PORT_STATS, num_stats);
1551                 goto fail;
1552         }
1553         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_PORT, PORT,
1554                                         ids, values, num_stats);
1555
1556         if (ret != NUM_PORT_STATS) {
1557                 printf("%d: expected %d stats, got return %d\n",
1558                                 __LINE__, NUM_PORT_STATS, ret);
1559                 goto fail;
1560         }
1561         static const char * const port_names[] = {
1562                 "port_2_rx",
1563                 "port_2_tx",
1564                 "port_2_drop",
1565                 "port_2_inflight",
1566                 "port_2_avg_pkt_cycles",
1567                 "port_2_credits",
1568                 "port_2_rx_ring_used",
1569                 "port_2_rx_ring_free",
1570                 "port_2_cq_ring_used",
1571                 "port_2_cq_ring_free",
1572                 "port_2_dequeue_calls",
1573                 "port_2_dequeues_returning_0",
1574                 "port_2_dequeues_returning_1-4",
1575                 "port_2_dequeues_returning_5-8",
1576                 "port_2_dequeues_returning_9-12",
1577                 "port_2_dequeues_returning_13-16",
1578                 "port_2_dequeues_returning_17-20",
1579                 "port_2_dequeues_returning_21-24",
1580                 "port_2_dequeues_returning_25-28",
1581                 "port_2_dequeues_returning_29-32",
1582                 "port_2_dequeues_returning_33-36",
1583         };
1584         uint64_t port_expected[] = {
1585                 0, /* rx */
1586                 NPKTS, /* tx */
1587                 0, /* drop */
1588                 NPKTS, /* inflight */
1589                 0, /* avg pkt cycles */
1590                 0, /* credits */
1591                 0, /* rx ring used */
1592                 4096, /* rx ring free */
1593                 NPKTS,  /* cq ring used */
1594                 25, /* cq ring free */
1595                 0, /* dequeue zero calls */
1596                 0, 0, 0, 0, 0, /* 10 dequeue buckets */
1597                 0, 0, 0, 0, 0,
1598         };
1599         uint64_t port_expected_zero[] = {
1600                 0, /* rx */
1601                 0, /* tx */
1602                 0, /* drop */
1603                 NPKTS, /* inflight */
1604                 0, /* avg pkt cycles */
1605                 0, /* credits */
1606                 0, /* rx ring used */
1607                 4096, /* rx ring free */
1608                 NPKTS,  /* cq ring used */
1609                 25, /* cq ring free */
1610                 0, /* dequeue zero calls */
1611                 0, 0, 0, 0, 0, /* 10 dequeue buckets */
1612                 0, 0, 0, 0, 0,
1613         };
1614         if (RTE_DIM(port_expected) != NUM_PORT_STATS ||
1615                         RTE_DIM(port_names) != NUM_PORT_STATS) {
1616                 printf("%d: port array of wrong size\n", __LINE__);
1617                 goto fail;
1618         }
1619
1620         int failed = 0;
1621         for (i = 0; (int)i < ret; i++) {
1622                 unsigned int id;
1623                 uint64_t val = rte_event_dev_xstats_by_name_get(evdev,
1624                                                                 port_names[i],
1625                                                                 &id);
1626                 if (id != i + PORT_OFF) {
1627                         printf("%d: %s id incorrect, expected %d got %d\n",
1628                                         __LINE__, port_names[i], i+PORT_OFF,
1629                                         id);
1630                         failed = 1;
1631                 }
1632                 if (val != port_expected[i]) {
1633                         printf("%d: %s value incorrect, expected %"PRIu64
1634                                 " got %d\n", __LINE__, port_names[i],
1635                                 port_expected[i], id);
1636                         failed = 1;
1637                 }
1638                 /* reset to zero */
1639                 int reset_ret = rte_event_dev_xstats_reset(evdev,
1640                                                 RTE_EVENT_DEV_XSTATS_PORT, PORT,
1641                                                 &id,
1642                                                 1);
1643                 if (reset_ret) {
1644                         printf("%d: failed to reset successfully\n", __LINE__);
1645                         failed = 1;
1646                 }
1647                 /* check value again */
1648                 val = rte_event_dev_xstats_by_name_get(evdev, port_names[i], 0);
1649                 if (val != port_expected_zero[i]) {
1650                         printf("%d: %s value incorrect, expected %"PRIu64
1651                                 " got %"PRIu64"\n", __LINE__, port_names[i],
1652                                 port_expected_zero[i], val);
1653                         failed = 1;
1654                 }
1655         };
1656         if (failed)
1657                 goto fail;
1658
1659 /* num queue stats */
1660 #define NUM_Q_STATS 17
1661 /* queue offset from start of the devices whole xstats.
1662  * This will break every time we add a statistic to a device/port/queue
1663  */
1664 #define QUEUE_OFF 90
1665         const uint32_t queue = 0;
1666         num_stats = rte_event_dev_xstats_names_get(evdev,
1667                                         RTE_EVENT_DEV_XSTATS_QUEUE, queue,
1668                                         xstats_names, ids, XSTATS_MAX);
1669         if (num_stats != NUM_Q_STATS) {
1670                 printf("%d: expected %d stats, got return %d\n",
1671                         __LINE__, NUM_Q_STATS, num_stats);
1672                 goto fail;
1673         }
1674         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE,
1675                                         queue, ids, values, num_stats);
1676         if (ret != NUM_Q_STATS) {
1677                 printf("%d: expected 21 stats, got return %d\n", __LINE__, ret);
1678                 goto fail;
1679         }
1680         static const char * const queue_names[] = {
1681                 "qid_0_rx",
1682                 "qid_0_tx",
1683                 "qid_0_drop",
1684                 "qid_0_inflight",
1685                 "qid_0_iq_size",
1686                 "qid_0_iq_0_used",
1687                 "qid_0_iq_1_used",
1688                 "qid_0_iq_2_used",
1689                 "qid_0_iq_3_used",
1690                 "qid_0_port_0_pinned_flows",
1691                 "qid_0_port_0_packets",
1692                 "qid_0_port_1_pinned_flows",
1693                 "qid_0_port_1_packets",
1694                 "qid_0_port_2_pinned_flows",
1695                 "qid_0_port_2_packets",
1696                 "qid_0_port_3_pinned_flows",
1697                 "qid_0_port_3_packets",
1698         };
1699         uint64_t queue_expected[] = {
1700                 7, /* rx */
1701                 7, /* tx */
1702                 0, /* drop */
1703                 7, /* inflight */
1704                 512, /* iq size */
1705                 0, /* iq 0 used */
1706                 0, /* iq 1 used */
1707                 0, /* iq 2 used */
1708                 0, /* iq 3 used */
1709                 /* QID-to-Port: pinned_flows, packets */
1710                 0, 0,
1711                 0, 0,
1712                 1, 7,
1713                 0, 0,
1714         };
1715         uint64_t queue_expected_zero[] = {
1716                 0, /* rx */
1717                 0, /* tx */
1718                 0, /* drop */
1719                 7, /* inflight */
1720                 512, /* iq size */
1721                 0, /* iq 0 used */
1722                 0, /* iq 1 used */
1723                 0, /* iq 2 used */
1724                 0, /* iq 3 used */
1725                 /* QID-to-Port: pinned_flows, packets */
1726                 0, 0,
1727                 0, 0,
1728                 1, 0,
1729                 0, 0,
1730         };
1731         if (RTE_DIM(queue_expected) != NUM_Q_STATS ||
1732                         RTE_DIM(queue_expected_zero) != NUM_Q_STATS ||
1733                         RTE_DIM(queue_names) != NUM_Q_STATS) {
1734                 printf("%d : queue array of wrong size\n", __LINE__);
1735                 goto fail;
1736         }
1737
1738         failed = 0;
1739         for (i = 0; (int)i < ret; i++) {
1740                 unsigned int id;
1741                 uint64_t val = rte_event_dev_xstats_by_name_get(evdev,
1742                                                                 queue_names[i],
1743                                                                 &id);
1744                 if (id != i + QUEUE_OFF) {
1745                         printf("%d: %s id incorrect, expected %d got %d\n",
1746                                         __LINE__, queue_names[i], i+QUEUE_OFF,
1747                                         id);
1748                         failed = 1;
1749                 }
1750                 if (val != queue_expected[i]) {
1751                         printf("%d: %d: %s value , expected %"PRIu64
1752                                 " got %"PRIu64"\n", i, __LINE__,
1753                                 queue_names[i], queue_expected[i], val);
1754                         failed = 1;
1755                 }
1756                 /* reset to zero */
1757                 int reset_ret = rte_event_dev_xstats_reset(evdev,
1758                                                 RTE_EVENT_DEV_XSTATS_QUEUE,
1759                                                 queue, &id, 1);
1760                 if (reset_ret) {
1761                         printf("%d: failed to reset successfully\n", __LINE__);
1762                         failed = 1;
1763                 }
1764                 /* check value again */
1765                 val = rte_event_dev_xstats_by_name_get(evdev, queue_names[i],
1766                                                         0);
1767                 if (val != queue_expected_zero[i]) {
1768                         printf("%d: %s value incorrect, expected %"PRIu64
1769                                 " got %"PRIu64"\n", __LINE__, queue_names[i],
1770                                 queue_expected_zero[i], val);
1771                         failed = 1;
1772                 }
1773         };
1774
1775         if (failed)
1776                 goto fail;
1777
1778         cleanup(t);
1779         return 0;
1780 fail:
1781         cleanup(t);
1782         return -1;
1783 }
1784
1785 static int
1786 ordered_reconfigure(struct test *t)
1787 {
1788         if (init(t, 1, 1) < 0 ||
1789                         create_ports(t, 1) < 0) {
1790                 printf("%d: Error initializing device\n", __LINE__);
1791                 return -1;
1792         }
1793
1794         const struct rte_event_queue_conf conf = {
1795                         .schedule_type = RTE_SCHED_TYPE_ORDERED,
1796                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1797                         .nb_atomic_flows = 1024,
1798                         .nb_atomic_order_sequences = 1024,
1799         };
1800
1801         if (rte_event_queue_setup(evdev, 0, &conf) < 0) {
1802                 printf("%d: error creating qid\n", __LINE__);
1803                 goto failed;
1804         }
1805
1806         if (rte_event_queue_setup(evdev, 0, &conf) < 0) {
1807                 printf("%d: error creating qid, for 2nd time\n", __LINE__);
1808                 goto failed;
1809         }
1810
1811         rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1812         if (rte_event_dev_start(evdev) < 0) {
1813                 printf("%d: Error with start call\n", __LINE__);
1814                 return -1;
1815         }
1816
1817         cleanup(t);
1818         return 0;
1819 failed:
1820         cleanup(t);
1821         return -1;
1822 }
1823
1824 static int
1825 qid_priorities(struct test *t)
1826 {
1827         /* Test works by having a CQ with enough empty space for all packets,
1828          * and enqueueing 3 packets to 3 QIDs. They must return based on the
1829          * priority of the QID, not the ingress order, to pass the test
1830          */
1831         unsigned int i;
1832         /* Create instance with 1 ports, and 3 qids */
1833         if (init(t, 3, 1) < 0 ||
1834                         create_ports(t, 1) < 0) {
1835                 printf("%d: Error initializing device\n", __LINE__);
1836                 return -1;
1837         }
1838
1839         for (i = 0; i < 3; i++) {
1840                 /* Create QID */
1841                 const struct rte_event_queue_conf conf = {
1842                         .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1843                         /* increase priority (0 == highest), as we go */
1844                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL - i,
1845                         .nb_atomic_flows = 1024,
1846                         .nb_atomic_order_sequences = 1024,
1847                 };
1848
1849                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
1850                         printf("%d: error creating qid %d\n", __LINE__, i);
1851                         return -1;
1852                 }
1853                 t->qid[i] = i;
1854         }
1855         t->nb_qids = i;
1856         /* map all QIDs to port */
1857         rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1858
1859         if (rte_event_dev_start(evdev) < 0) {
1860                 printf("%d: Error with start call\n", __LINE__);
1861                 return -1;
1862         }
1863
1864         /* enqueue 3 packets, setting seqn and QID to check priority */
1865         for (i = 0; i < 3; i++) {
1866                 struct rte_event ev;
1867                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1868                 if (!arp) {
1869                         printf("%d: gen of pkt failed\n", __LINE__);
1870                         return -1;
1871                 }
1872                 ev.queue_id = t->qid[i];
1873                 ev.op = RTE_EVENT_OP_NEW;
1874                 ev.mbuf = arp;
1875                 arp->seqn = i;
1876
1877                 int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
1878                 if (err != 1) {
1879                         printf("%d: Failed to enqueue\n", __LINE__);
1880                         return -1;
1881                 }
1882         }
1883
1884         rte_service_run_iter_on_app_lcore(t->service_id, 1);
1885
1886         /* dequeue packets, verify priority was upheld */
1887         struct rte_event ev[32];
1888         uint32_t deq_pkts =
1889                 rte_event_dequeue_burst(evdev, t->port[0], ev, 32, 0);
1890         if (deq_pkts != 3) {
1891                 printf("%d: failed to deq packets\n", __LINE__);
1892                 rte_event_dev_dump(evdev, stdout);
1893                 return -1;
1894         }
1895         for (i = 0; i < 3; i++) {
1896                 if (ev[i].mbuf->seqn != 2-i) {
1897                         printf(
1898                                 "%d: qid priority test: seqn %d incorrectly prioritized\n",
1899                                         __LINE__, i);
1900                 }
1901         }
1902
1903         cleanup(t);
1904         return 0;
1905 }
1906
1907 static int
1908 load_balancing(struct test *t)
1909 {
1910         const int rx_enq = 0;
1911         int err;
1912         uint32_t i;
1913
1914         if (init(t, 1, 4) < 0 ||
1915                         create_ports(t, 4) < 0 ||
1916                         create_atomic_qids(t, 1) < 0) {
1917                 printf("%d: Error initializing device\n", __LINE__);
1918                 return -1;
1919         }
1920
1921         for (i = 0; i < 3; i++) {
1922                 /* map port 1 - 3 inclusive */
1923                 if (rte_event_port_link(evdev, t->port[i+1], &t->qid[0],
1924                                 NULL, 1) != 1) {
1925                         printf("%d: error mapping qid to port %d\n",
1926                                         __LINE__, i);
1927                         return -1;
1928                 }
1929         }
1930
1931         if (rte_event_dev_start(evdev) < 0) {
1932                 printf("%d: Error with start call\n", __LINE__);
1933                 return -1;
1934         }
1935
1936         /************** FORWARD ****************/
1937         /*
1938          * Create a set of flows that test the load-balancing operation of the
1939          * implementation. Fill CQ 0 and 1 with flows 0 and 1, and test
1940          * with a new flow, which should be sent to the 3rd mapped CQ
1941          */
1942         static uint32_t flows[] = {0, 1, 1, 0, 0, 2, 2, 0, 2};
1943
1944         for (i = 0; i < RTE_DIM(flows); i++) {
1945                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1946                 if (!arp) {
1947                         printf("%d: gen of pkt failed\n", __LINE__);
1948                         return -1;
1949                 }
1950
1951                 struct rte_event ev = {
1952                                 .op = RTE_EVENT_OP_NEW,
1953                                 .queue_id = t->qid[0],
1954                                 .flow_id = flows[i],
1955                                 .mbuf = arp,
1956                 };
1957                 /* generate pkt and enqueue */
1958                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
1959                 if (err < 0) {
1960                         printf("%d: Failed to enqueue\n", __LINE__);
1961                         return -1;
1962                 }
1963         }
1964
1965         rte_service_run_iter_on_app_lcore(t->service_id, 1);
1966
1967         struct test_event_dev_stats stats;
1968         err = test_event_dev_stats_get(evdev, &stats);
1969         if (err) {
1970                 printf("%d: failed to get stats\n", __LINE__);
1971                 return -1;
1972         }
1973
1974         if (stats.port_inflight[1] != 4) {
1975                 printf("%d:%s: port 1 inflight not correct\n", __LINE__,
1976                                 __func__);
1977                 return -1;
1978         }
1979         if (stats.port_inflight[2] != 2) {
1980                 printf("%d:%s: port 2 inflight not correct\n", __LINE__,
1981                                 __func__);
1982                 return -1;
1983         }
1984         if (stats.port_inflight[3] != 3) {
1985                 printf("%d:%s: port 3 inflight not correct\n", __LINE__,
1986                                 __func__);
1987                 return -1;
1988         }
1989
1990         cleanup(t);
1991         return 0;
1992 }
1993
1994 static int
1995 load_balancing_history(struct test *t)
1996 {
1997         struct test_event_dev_stats stats = {0};
1998         const int rx_enq = 0;
1999         int err;
2000         uint32_t i;
2001
2002         /* Create instance with 1 atomic QID going to 3 ports + 1 prod port */
2003         if (init(t, 1, 4) < 0 ||
2004                         create_ports(t, 4) < 0 ||
2005                         create_atomic_qids(t, 1) < 0)
2006                 return -1;
2007
2008         /* CQ mapping to QID */
2009         if (rte_event_port_link(evdev, t->port[1], &t->qid[0], NULL, 1) != 1) {
2010                 printf("%d: error mapping port 1 qid\n", __LINE__);
2011                 return -1;
2012         }
2013         if (rte_event_port_link(evdev, t->port[2], &t->qid[0], NULL, 1) != 1) {
2014                 printf("%d: error mapping port 2 qid\n", __LINE__);
2015                 return -1;
2016         }
2017         if (rte_event_port_link(evdev, t->port[3], &t->qid[0], NULL, 1) != 1) {
2018                 printf("%d: error mapping port 3 qid\n", __LINE__);
2019                 return -1;
2020         }
2021         if (rte_event_dev_start(evdev) < 0) {
2022                 printf("%d: Error with start call\n", __LINE__);
2023                 return -1;
2024         }
2025
2026         /*
2027          * Create a set of flows that test the load-balancing operation of the
2028          * implementation. Fill CQ 0, 1 and 2 with flows 0, 1 and 2, drop
2029          * the packet from CQ 0, send in a new set of flows. Ensure that:
2030          *  1. The new flow 3 gets into the empty CQ0
2031          *  2. packets for existing flow gets added into CQ1
2032          *  3. Next flow 0 pkt is now onto CQ2, since CQ0 and CQ1 now contain
2033          *     more outstanding pkts
2034          *
2035          *  This test makes sure that when a flow ends (i.e. all packets
2036          *  have been completed for that flow), that the flow can be moved
2037          *  to a different CQ when new packets come in for that flow.
2038          */
2039         static uint32_t flows1[] = {0, 1, 1, 2};
2040
2041         for (i = 0; i < RTE_DIM(flows1); i++) {
2042                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2043                 struct rte_event ev = {
2044                                 .flow_id = flows1[i],
2045                                 .op = RTE_EVENT_OP_NEW,
2046                                 .queue_id = t->qid[0],
2047                                 .event_type = RTE_EVENT_TYPE_CPU,
2048                                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
2049                                 .mbuf = arp
2050                 };
2051
2052                 if (!arp) {
2053                         printf("%d: gen of pkt failed\n", __LINE__);
2054                         return -1;
2055                 }
2056                 arp->hash.rss = flows1[i];
2057                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2058                 if (err < 0) {
2059                         printf("%d: Failed to enqueue\n", __LINE__);
2060                         return -1;
2061                 }
2062         }
2063
2064         /* call the scheduler */
2065         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2066
2067         /* Dequeue the flow 0 packet from port 1, so that we can then drop */
2068         struct rte_event ev;
2069         if (!rte_event_dequeue_burst(evdev, t->port[1], &ev, 1, 0)) {
2070                 printf("%d: failed to dequeue\n", __LINE__);
2071                 return -1;
2072         }
2073         if (ev.mbuf->hash.rss != flows1[0]) {
2074                 printf("%d: unexpected flow received\n", __LINE__);
2075                 return -1;
2076         }
2077
2078         /* drop the flow 0 packet from port 1 */
2079         rte_event_enqueue_burst(evdev, t->port[1], &release_ev, 1);
2080
2081         /* call the scheduler */
2082         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2083
2084         /*
2085          * Set up the next set of flows, first a new flow to fill up
2086          * CQ 0, so that the next flow 0 packet should go to CQ2
2087          */
2088         static uint32_t flows2[] = { 3, 3, 3, 1, 1, 0 };
2089
2090         for (i = 0; i < RTE_DIM(flows2); i++) {
2091                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2092                 struct rte_event ev = {
2093                                 .flow_id = flows2[i],
2094                                 .op = RTE_EVENT_OP_NEW,
2095                                 .queue_id = t->qid[0],
2096                                 .event_type = RTE_EVENT_TYPE_CPU,
2097                                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
2098                                 .mbuf = arp
2099                 };
2100
2101                 if (!arp) {
2102                         printf("%d: gen of pkt failed\n", __LINE__);
2103                         return -1;
2104                 }
2105                 arp->hash.rss = flows2[i];
2106
2107                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2108                 if (err < 0) {
2109                         printf("%d: Failed to enqueue\n", __LINE__);
2110                         return -1;
2111                 }
2112         }
2113
2114         /* schedule */
2115         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2116
2117         err = test_event_dev_stats_get(evdev, &stats);
2118         if (err) {
2119                 printf("%d:failed to get stats\n", __LINE__);
2120                 return -1;
2121         }
2122
2123         /*
2124          * Now check the resulting inflights on each port.
2125          */
2126         if (stats.port_inflight[1] != 3) {
2127                 printf("%d:%s: port 1 inflight not correct\n", __LINE__,
2128                                 __func__);
2129                 printf("Inflights, ports 1, 2, 3: %u, %u, %u\n",
2130                                 (unsigned int)stats.port_inflight[1],
2131                                 (unsigned int)stats.port_inflight[2],
2132                                 (unsigned int)stats.port_inflight[3]);
2133                 return -1;
2134         }
2135         if (stats.port_inflight[2] != 4) {
2136                 printf("%d:%s: port 2 inflight not correct\n", __LINE__,
2137                                 __func__);
2138                 printf("Inflights, ports 1, 2, 3: %u, %u, %u\n",
2139                                 (unsigned int)stats.port_inflight[1],
2140                                 (unsigned int)stats.port_inflight[2],
2141                                 (unsigned int)stats.port_inflight[3]);
2142                 return -1;
2143         }
2144         if (stats.port_inflight[3] != 2) {
2145                 printf("%d:%s: port 3 inflight not correct\n", __LINE__,
2146                                 __func__);
2147                 printf("Inflights, ports 1, 2, 3: %u, %u, %u\n",
2148                                 (unsigned int)stats.port_inflight[1],
2149                                 (unsigned int)stats.port_inflight[2],
2150                                 (unsigned int)stats.port_inflight[3]);
2151                 return -1;
2152         }
2153
2154         for (i = 1; i <= 3; i++) {
2155                 struct rte_event ev;
2156                 while (rte_event_dequeue_burst(evdev, i, &ev, 1, 0))
2157                         rte_event_enqueue_burst(evdev, i, &release_ev, 1);
2158         }
2159         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2160
2161         cleanup(t);
2162         return 0;
2163 }
2164
2165 static int
2166 invalid_qid(struct test *t)
2167 {
2168         struct test_event_dev_stats stats;
2169         const int rx_enq = 0;
2170         int err;
2171         uint32_t i;
2172
2173         if (init(t, 1, 4) < 0 ||
2174                         create_ports(t, 4) < 0 ||
2175                         create_atomic_qids(t, 1) < 0) {
2176                 printf("%d: Error initializing device\n", __LINE__);
2177                 return -1;
2178         }
2179
2180         /* CQ mapping to QID */
2181         for (i = 0; i < 4; i++) {
2182                 err = rte_event_port_link(evdev, t->port[i], &t->qid[0],
2183                                 NULL, 1);
2184                 if (err != 1) {
2185                         printf("%d: error mapping port 1 qid\n", __LINE__);
2186                         return -1;
2187                 }
2188         }
2189
2190         if (rte_event_dev_start(evdev) < 0) {
2191                 printf("%d: Error with start call\n", __LINE__);
2192                 return -1;
2193         }
2194
2195         /*
2196          * Send in a packet with an invalid qid to the scheduler.
2197          * We should see the packed enqueued OK, but the inflights for
2198          * that packet should not be incremented, and the rx_dropped
2199          * should be incremented.
2200          */
2201         static uint32_t flows1[] = {20};
2202
2203         for (i = 0; i < RTE_DIM(flows1); i++) {
2204                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2205                 if (!arp) {
2206                         printf("%d: gen of pkt failed\n", __LINE__);
2207                         return -1;
2208                 }
2209
2210                 struct rte_event ev = {
2211                                 .op = RTE_EVENT_OP_NEW,
2212                                 .queue_id = t->qid[0] + flows1[i],
2213                                 .flow_id = i,
2214                                 .mbuf = arp,
2215                 };
2216                 /* generate pkt and enqueue */
2217                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2218                 if (err < 0) {
2219                         printf("%d: Failed to enqueue\n", __LINE__);
2220                         return -1;
2221                 }
2222         }
2223
2224         /* call the scheduler */
2225         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2226
2227         err = test_event_dev_stats_get(evdev, &stats);
2228         if (err) {
2229                 printf("%d: failed to get stats\n", __LINE__);
2230                 return -1;
2231         }
2232
2233         /*
2234          * Now check the resulting inflights on the port, and the rx_dropped.
2235          */
2236         if (stats.port_inflight[0] != 0) {
2237                 printf("%d:%s: port 1 inflight count not correct\n", __LINE__,
2238                                 __func__);
2239                 rte_event_dev_dump(evdev, stdout);
2240                 return -1;
2241         }
2242         if (stats.port_rx_dropped[0] != 1) {
2243                 printf("%d:%s: port 1 drops\n", __LINE__, __func__);
2244                 rte_event_dev_dump(evdev, stdout);
2245                 return -1;
2246         }
2247         /* each packet drop should only be counted in one place - port or dev */
2248         if (stats.rx_dropped != 0) {
2249                 printf("%d:%s: port 1 dropped count not correct\n", __LINE__,
2250                                 __func__);
2251                 rte_event_dev_dump(evdev, stdout);
2252                 return -1;
2253         }
2254
2255         cleanup(t);
2256         return 0;
2257 }
2258
2259 static int
2260 single_packet(struct test *t)
2261 {
2262         const uint32_t MAGIC_SEQN = 7321;
2263         struct rte_event ev;
2264         struct test_event_dev_stats stats;
2265         const int rx_enq = 0;
2266         const int wrk_enq = 2;
2267         int err;
2268
2269         /* Create instance with 4 ports */
2270         if (init(t, 1, 4) < 0 ||
2271                         create_ports(t, 4) < 0 ||
2272                         create_atomic_qids(t, 1) < 0) {
2273                 printf("%d: Error initializing device\n", __LINE__);
2274                 return -1;
2275         }
2276
2277         /* CQ mapping to QID */
2278         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
2279         if (err != 1) {
2280                 printf("%d: error mapping lb qid\n", __LINE__);
2281                 cleanup(t);
2282                 return -1;
2283         }
2284
2285         if (rte_event_dev_start(evdev) < 0) {
2286                 printf("%d: Error with start call\n", __LINE__);
2287                 return -1;
2288         }
2289
2290         /************** Gen pkt and enqueue ****************/
2291         struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2292         if (!arp) {
2293                 printf("%d: gen of pkt failed\n", __LINE__);
2294                 return -1;
2295         }
2296
2297         ev.op = RTE_EVENT_OP_NEW;
2298         ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
2299         ev.mbuf = arp;
2300         ev.queue_id = 0;
2301         ev.flow_id = 3;
2302         arp->seqn = MAGIC_SEQN;
2303
2304         err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2305         if (err < 0) {
2306                 printf("%d: Failed to enqueue\n", __LINE__);
2307                 return -1;
2308         }
2309
2310         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2311
2312         err = test_event_dev_stats_get(evdev, &stats);
2313         if (err) {
2314                 printf("%d: failed to get stats\n", __LINE__);
2315                 return -1;
2316         }
2317
2318         if (stats.rx_pkts != 1 ||
2319                         stats.tx_pkts != 1 ||
2320                         stats.port_inflight[wrk_enq] != 1) {
2321                 printf("%d: Sched core didn't handle pkt as expected\n",
2322                                 __LINE__);
2323                 rte_event_dev_dump(evdev, stdout);
2324                 return -1;
2325         }
2326
2327         uint32_t deq_pkts;
2328
2329         deq_pkts = rte_event_dequeue_burst(evdev, t->port[wrk_enq], &ev, 1, 0);
2330         if (deq_pkts < 1) {
2331                 printf("%d: Failed to deq\n", __LINE__);
2332                 return -1;
2333         }
2334
2335         err = test_event_dev_stats_get(evdev, &stats);
2336         if (err) {
2337                 printf("%d: failed to get stats\n", __LINE__);
2338                 return -1;
2339         }
2340
2341         err = test_event_dev_stats_get(evdev, &stats);
2342         if (ev.mbuf->seqn != MAGIC_SEQN) {
2343                 printf("%d: magic sequence number not dequeued\n", __LINE__);
2344                 return -1;
2345         }
2346
2347         rte_pktmbuf_free(ev.mbuf);
2348         err = rte_event_enqueue_burst(evdev, t->port[wrk_enq], &release_ev, 1);
2349         if (err < 0) {
2350                 printf("%d: Failed to enqueue\n", __LINE__);
2351                 return -1;
2352         }
2353         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2354
2355         err = test_event_dev_stats_get(evdev, &stats);
2356         if (stats.port_inflight[wrk_enq] != 0) {
2357                 printf("%d: port inflight not correct\n", __LINE__);
2358                 return -1;
2359         }
2360
2361         cleanup(t);
2362         return 0;
2363 }
2364
2365 static int
2366 inflight_counts(struct test *t)
2367 {
2368         struct rte_event ev;
2369         struct test_event_dev_stats stats;
2370         const int rx_enq = 0;
2371         const int p1 = 1;
2372         const int p2 = 2;
2373         int err;
2374         int i;
2375
2376         /* Create instance with 4 ports */
2377         if (init(t, 2, 3) < 0 ||
2378                         create_ports(t, 3) < 0 ||
2379                         create_atomic_qids(t, 2) < 0) {
2380                 printf("%d: Error initializing device\n", __LINE__);
2381                 return -1;
2382         }
2383
2384         /* CQ mapping to QID */
2385         err = rte_event_port_link(evdev, t->port[p1], &t->qid[0], NULL, 1);
2386         if (err != 1) {
2387                 printf("%d: error mapping lb qid\n", __LINE__);
2388                 cleanup(t);
2389                 return -1;
2390         }
2391         err = rte_event_port_link(evdev, t->port[p2], &t->qid[1], NULL, 1);
2392         if (err != 1) {
2393                 printf("%d: error mapping lb qid\n", __LINE__);
2394                 cleanup(t);
2395                 return -1;
2396         }
2397
2398         if (rte_event_dev_start(evdev) < 0) {
2399                 printf("%d: Error with start call\n", __LINE__);
2400                 return -1;
2401         }
2402
2403         /************** FORWARD ****************/
2404 #define QID1_NUM 5
2405         for (i = 0; i < QID1_NUM; i++) {
2406                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2407
2408                 if (!arp) {
2409                         printf("%d: gen of pkt failed\n", __LINE__);
2410                         goto err;
2411                 }
2412
2413                 ev.queue_id =  t->qid[0];
2414                 ev.op = RTE_EVENT_OP_NEW;
2415                 ev.mbuf = arp;
2416                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2417                 if (err != 1) {
2418                         printf("%d: Failed to enqueue\n", __LINE__);
2419                         goto err;
2420                 }
2421         }
2422 #define QID2_NUM 3
2423         for (i = 0; i < QID2_NUM; i++) {
2424                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2425
2426                 if (!arp) {
2427                         printf("%d: gen of pkt failed\n", __LINE__);
2428                         goto err;
2429                 }
2430                 ev.queue_id =  t->qid[1];
2431                 ev.op = RTE_EVENT_OP_NEW;
2432                 ev.mbuf = arp;
2433                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2434                 if (err != 1) {
2435                         printf("%d: Failed to enqueue\n", __LINE__);
2436                         goto err;
2437                 }
2438         }
2439
2440         /* schedule */
2441         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2442
2443         err = test_event_dev_stats_get(evdev, &stats);
2444         if (err) {
2445                 printf("%d: failed to get stats\n", __LINE__);
2446                 goto err;
2447         }
2448
2449         if (stats.rx_pkts != QID1_NUM + QID2_NUM ||
2450                         stats.tx_pkts != QID1_NUM + QID2_NUM) {
2451                 printf("%d: Sched core didn't handle pkt as expected\n",
2452                                 __LINE__);
2453                 goto err;
2454         }
2455
2456         if (stats.port_inflight[p1] != QID1_NUM) {
2457                 printf("%d: %s port 1 inflight not correct\n", __LINE__,
2458                                 __func__);
2459                 goto err;
2460         }
2461         if (stats.port_inflight[p2] != QID2_NUM) {
2462                 printf("%d: %s port 2 inflight not correct\n", __LINE__,
2463                                 __func__);
2464                 goto err;
2465         }
2466
2467         /************** DEQUEUE INFLIGHT COUNT CHECKS  ****************/
2468         /* port 1 */
2469         struct rte_event events[QID1_NUM + QID2_NUM];
2470         uint32_t deq_pkts = rte_event_dequeue_burst(evdev, t->port[p1], events,
2471                         RTE_DIM(events), 0);
2472
2473         if (deq_pkts != QID1_NUM) {
2474                 printf("%d: Port 1: DEQUEUE inflight failed\n", __LINE__);
2475                 goto err;
2476         }
2477         err = test_event_dev_stats_get(evdev, &stats);
2478         if (stats.port_inflight[p1] != QID1_NUM) {
2479                 printf("%d: port 1 inflight decrement after DEQ != 0\n",
2480                                 __LINE__);
2481                 goto err;
2482         }
2483         for (i = 0; i < QID1_NUM; i++) {
2484                 err = rte_event_enqueue_burst(evdev, t->port[p1], &release_ev,
2485                                 1);
2486                 if (err != 1) {
2487                         printf("%d: %s rte enqueue of inf release failed\n",
2488                                 __LINE__, __func__);
2489                         goto err;
2490                 }
2491         }
2492
2493         /*
2494          * As the scheduler core decrements inflights, it needs to run to
2495          * process packets to act on the drop messages
2496          */
2497         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2498
2499         err = test_event_dev_stats_get(evdev, &stats);
2500         if (stats.port_inflight[p1] != 0) {
2501                 printf("%d: port 1 inflight NON NULL after DROP\n", __LINE__);
2502                 goto err;
2503         }
2504
2505         /* port2 */
2506         deq_pkts = rte_event_dequeue_burst(evdev, t->port[p2], events,
2507                         RTE_DIM(events), 0);
2508         if (deq_pkts != QID2_NUM) {
2509                 printf("%d: Port 2: DEQUEUE inflight failed\n", __LINE__);
2510                 goto err;
2511         }
2512         err = test_event_dev_stats_get(evdev, &stats);
2513         if (stats.port_inflight[p2] != QID2_NUM) {
2514                 printf("%d: port 1 inflight decrement after DEQ != 0\n",
2515                                 __LINE__);
2516                 goto err;
2517         }
2518         for (i = 0; i < QID2_NUM; i++) {
2519                 err = rte_event_enqueue_burst(evdev, t->port[p2], &release_ev,
2520                                 1);
2521                 if (err != 1) {
2522                         printf("%d: %s rte enqueue of inf release failed\n",
2523                                 __LINE__, __func__);
2524                         goto err;
2525                 }
2526         }
2527
2528         /*
2529          * As the scheduler core decrements inflights, it needs to run to
2530          * process packets to act on the drop messages
2531          */
2532         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2533
2534         err = test_event_dev_stats_get(evdev, &stats);
2535         if (stats.port_inflight[p2] != 0) {
2536                 printf("%d: port 2 inflight NON NULL after DROP\n", __LINE__);
2537                 goto err;
2538         }
2539         cleanup(t);
2540         return 0;
2541
2542 err:
2543         rte_event_dev_dump(evdev, stdout);
2544         cleanup(t);
2545         return -1;
2546 }
2547
2548 static int
2549 parallel_basic(struct test *t, int check_order)
2550 {
2551         const uint8_t rx_port = 0;
2552         const uint8_t w1_port = 1;
2553         const uint8_t w3_port = 3;
2554         const uint8_t tx_port = 4;
2555         int err;
2556         int i;
2557         uint32_t deq_pkts, j;
2558         struct rte_mbuf *mbufs[3];
2559         struct rte_mbuf *mbufs_out[3] = { 0 };
2560         const uint32_t MAGIC_SEQN = 1234;
2561
2562         /* Create instance with 4 ports */
2563         if (init(t, 2, tx_port + 1) < 0 ||
2564                         create_ports(t, tx_port + 1) < 0 ||
2565                         (check_order ?  create_ordered_qids(t, 1) :
2566                                 create_unordered_qids(t, 1)) < 0 ||
2567                         create_directed_qids(t, 1, &tx_port)) {
2568                 printf("%d: Error initializing device\n", __LINE__);
2569                 return -1;
2570         }
2571
2572         /*
2573          * CQ mapping to QID
2574          * We need three ports, all mapped to the same ordered qid0. Then we'll
2575          * take a packet out to each port, re-enqueue in reverse order,
2576          * then make sure the reordering has taken place properly when we
2577          * dequeue from the tx_port.
2578          *
2579          * Simplified test setup diagram:
2580          *
2581          * rx_port        w1_port
2582          *        \     /         \
2583          *         qid0 - w2_port - qid1
2584          *              \         /     \
2585          *                w3_port        tx_port
2586          */
2587         /* CQ mapping to QID for LB ports (directed mapped on create) */
2588         for (i = w1_port; i <= w3_port; i++) {
2589                 err = rte_event_port_link(evdev, t->port[i], &t->qid[0], NULL,
2590                                 1);
2591                 if (err != 1) {
2592                         printf("%d: error mapping lb qid\n", __LINE__);
2593                         cleanup(t);
2594                         return -1;
2595                 }
2596         }
2597
2598         if (rte_event_dev_start(evdev) < 0) {
2599                 printf("%d: Error with start call\n", __LINE__);
2600                 return -1;
2601         }
2602
2603         /* Enqueue 3 packets to the rx port */
2604         for (i = 0; i < 3; i++) {
2605                 struct rte_event ev;
2606                 mbufs[i] = rte_gen_arp(0, t->mbuf_pool);
2607                 if (!mbufs[i]) {
2608                         printf("%d: gen of pkt failed\n", __LINE__);
2609                         return -1;
2610                 }
2611
2612                 ev.queue_id = t->qid[0];
2613                 ev.op = RTE_EVENT_OP_NEW;
2614                 ev.mbuf = mbufs[i];
2615                 mbufs[i]->seqn = MAGIC_SEQN + i;
2616
2617                 /* generate pkt and enqueue */
2618                 err = rte_event_enqueue_burst(evdev, t->port[rx_port], &ev, 1);
2619                 if (err != 1) {
2620                         printf("%d: Failed to enqueue pkt %u, retval = %u\n",
2621                                         __LINE__, i, err);
2622                         return -1;
2623                 }
2624         }
2625
2626         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2627
2628         /* use extra slot to make logic in loops easier */
2629         struct rte_event deq_ev[w3_port + 1];
2630
2631         /* Dequeue the 3 packets, one from each worker port */
2632         for (i = w1_port; i <= w3_port; i++) {
2633                 deq_pkts = rte_event_dequeue_burst(evdev, t->port[i],
2634                                 &deq_ev[i], 1, 0);
2635                 if (deq_pkts != 1) {
2636                         printf("%d: Failed to deq\n", __LINE__);
2637                         rte_event_dev_dump(evdev, stdout);
2638                         return -1;
2639                 }
2640         }
2641
2642         /* Enqueue each packet in reverse order, flushing after each one */
2643         for (i = w3_port; i >= w1_port; i--) {
2644
2645                 deq_ev[i].op = RTE_EVENT_OP_FORWARD;
2646                 deq_ev[i].queue_id = t->qid[1];
2647                 err = rte_event_enqueue_burst(evdev, t->port[i], &deq_ev[i], 1);
2648                 if (err != 1) {
2649                         printf("%d: Failed to enqueue\n", __LINE__);
2650                         return -1;
2651                 }
2652         }
2653         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2654
2655         /* dequeue from the tx ports, we should get 3 packets */
2656         deq_pkts = rte_event_dequeue_burst(evdev, t->port[tx_port], deq_ev,
2657                         3, 0);
2658
2659         /* Check to see if we've got all 3 packets */
2660         if (deq_pkts != 3) {
2661                 printf("%d: expected 3 pkts at tx port got %d from port %d\n",
2662                         __LINE__, deq_pkts, tx_port);
2663                 rte_event_dev_dump(evdev, stdout);
2664                 return 1;
2665         }
2666
2667         /* Check to see if the sequence numbers are in expected order */
2668         if (check_order) {
2669                 for (j = 0 ; j < deq_pkts ; j++) {
2670                         if (deq_ev[j].mbuf->seqn != MAGIC_SEQN + j) {
2671                                 printf(
2672                                         "%d: Incorrect sequence number(%d) from port %d\n",
2673                                         __LINE__, mbufs_out[j]->seqn, tx_port);
2674                                 return -1;
2675                         }
2676                 }
2677         }
2678
2679         /* Destroy the instance */
2680         cleanup(t);
2681         return 0;
2682 }
2683
2684 static int
2685 ordered_basic(struct test *t)
2686 {
2687         return parallel_basic(t, 1);
2688 }
2689
2690 static int
2691 unordered_basic(struct test *t)
2692 {
2693         return parallel_basic(t, 0);
2694 }
2695
2696 static int
2697 holb(struct test *t) /* test to check we avoid basic head-of-line blocking */
2698 {
2699         const struct rte_event new_ev = {
2700                         .op = RTE_EVENT_OP_NEW
2701                         /* all other fields zero */
2702         };
2703         struct rte_event ev = new_ev;
2704         unsigned int rx_port = 0; /* port we get the first flow on */
2705         char rx_port_used_stat[64];
2706         char rx_port_free_stat[64];
2707         char other_port_used_stat[64];
2708
2709         if (init(t, 1, 2) < 0 ||
2710                         create_ports(t, 2) < 0 ||
2711                         create_atomic_qids(t, 1) < 0) {
2712                 printf("%d: Error initializing device\n", __LINE__);
2713                 return -1;
2714         }
2715         int nb_links = rte_event_port_link(evdev, t->port[1], NULL, NULL, 0);
2716         if (rte_event_port_link(evdev, t->port[0], NULL, NULL, 0) != 1 ||
2717                         nb_links != 1) {
2718                 printf("%d: Error links queue to ports\n", __LINE__);
2719                 goto err;
2720         }
2721         if (rte_event_dev_start(evdev) < 0) {
2722                 printf("%d: Error with start call\n", __LINE__);
2723                 goto err;
2724         }
2725
2726         /* send one packet and see where it goes, port 0 or 1 */
2727         if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2728                 printf("%d: Error doing first enqueue\n", __LINE__);
2729                 goto err;
2730         }
2731         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2732
2733         if (rte_event_dev_xstats_by_name_get(evdev, "port_0_cq_ring_used", NULL)
2734                         != 1)
2735                 rx_port = 1;
2736
2737         snprintf(rx_port_used_stat, sizeof(rx_port_used_stat),
2738                         "port_%u_cq_ring_used", rx_port);
2739         snprintf(rx_port_free_stat, sizeof(rx_port_free_stat),
2740                         "port_%u_cq_ring_free", rx_port);
2741         snprintf(other_port_used_stat, sizeof(other_port_used_stat),
2742                         "port_%u_cq_ring_used", rx_port ^ 1);
2743         if (rte_event_dev_xstats_by_name_get(evdev, rx_port_used_stat, NULL)
2744                         != 1) {
2745                 printf("%d: Error, first event not scheduled\n", __LINE__);
2746                 goto err;
2747         }
2748
2749         /* now fill up the rx port's queue with one flow to cause HOLB */
2750         do {
2751                 ev = new_ev;
2752                 if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2753                         printf("%d: Error with enqueue\n", __LINE__);
2754                         goto err;
2755                 }
2756                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
2757         } while (rte_event_dev_xstats_by_name_get(evdev,
2758                                 rx_port_free_stat, NULL) != 0);
2759
2760         /* one more packet, which needs to stay in IQ - i.e. HOLB */
2761         ev = new_ev;
2762         if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2763                 printf("%d: Error with enqueue\n", __LINE__);
2764                 goto err;
2765         }
2766         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2767
2768         /* check that the other port still has an empty CQ */
2769         if (rte_event_dev_xstats_by_name_get(evdev, other_port_used_stat, NULL)
2770                         != 0) {
2771                 printf("%d: Error, second port CQ is not empty\n", __LINE__);
2772                 goto err;
2773         }
2774         /* check IQ now has one packet */
2775         if (rte_event_dev_xstats_by_name_get(evdev, "qid_0_iq_0_used", NULL)
2776                         != 1) {
2777                 printf("%d: Error, QID does not have exactly 1 packet\n",
2778                         __LINE__);
2779                 goto err;
2780         }
2781
2782         /* send another flow, which should pass the other IQ entry */
2783         ev = new_ev;
2784         ev.flow_id = 1;
2785         if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2786                 printf("%d: Error with enqueue\n", __LINE__);
2787                 goto err;
2788         }
2789         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2790
2791         if (rte_event_dev_xstats_by_name_get(evdev, other_port_used_stat, NULL)
2792                         != 1) {
2793                 printf("%d: Error, second flow did not pass out first\n",
2794                         __LINE__);
2795                 goto err;
2796         }
2797
2798         if (rte_event_dev_xstats_by_name_get(evdev, "qid_0_iq_0_used", NULL)
2799                         != 1) {
2800                 printf("%d: Error, QID does not have exactly 1 packet\n",
2801                         __LINE__);
2802                 goto err;
2803         }
2804         cleanup(t);
2805         return 0;
2806 err:
2807         rte_event_dev_dump(evdev, stdout);
2808         cleanup(t);
2809         return -1;
2810 }
2811
2812 static int
2813 worker_loopback_worker_fn(void *arg)
2814 {
2815         struct test *t = arg;
2816         uint8_t port = t->port[1];
2817         int count = 0;
2818         int enqd;
2819
2820         /*
2821          * Takes packets from the input port and then loops them back through
2822          * the Eventdev. Each packet gets looped through QIDs 0-8, 16 times
2823          * so each packet goes through 8*16 = 128 times.
2824          */
2825         printf("%d: \tWorker function started\n", __LINE__);
2826         while (count < NUM_PACKETS) {
2827 #define BURST_SIZE 32
2828                 struct rte_event ev[BURST_SIZE];
2829                 uint16_t i, nb_rx = rte_event_dequeue_burst(evdev, port, ev,
2830                                 BURST_SIZE, 0);
2831                 if (nb_rx == 0) {
2832                         rte_pause();
2833                         continue;
2834                 }
2835
2836                 for (i = 0; i < nb_rx; i++) {
2837                         ev[i].queue_id++;
2838                         if (ev[i].queue_id != 8) {
2839                                 ev[i].op = RTE_EVENT_OP_FORWARD;
2840                                 enqd = rte_event_enqueue_burst(evdev, port,
2841                                                 &ev[i], 1);
2842                                 if (enqd != 1) {
2843                                         printf("%d: Can't enqueue FWD!!\n",
2844                                                         __LINE__);
2845                                         return -1;
2846                                 }
2847                                 continue;
2848                         }
2849
2850                         ev[i].queue_id = 0;
2851                         ev[i].mbuf->udata64++;
2852                         if (ev[i].mbuf->udata64 != 16) {
2853                                 ev[i].op = RTE_EVENT_OP_FORWARD;
2854                                 enqd = rte_event_enqueue_burst(evdev, port,
2855                                                 &ev[i], 1);
2856                                 if (enqd != 1) {
2857                                         printf("%d: Can't enqueue FWD!!\n",
2858                                                         __LINE__);
2859                                         return -1;
2860                                 }
2861                                 continue;
2862                         }
2863                         /* we have hit 16 iterations through system - drop */
2864                         rte_pktmbuf_free(ev[i].mbuf);
2865                         count++;
2866                         ev[i].op = RTE_EVENT_OP_RELEASE;
2867                         enqd = rte_event_enqueue_burst(evdev, port, &ev[i], 1);
2868                         if (enqd != 1) {
2869                                 printf("%d drop enqueue failed\n", __LINE__);
2870                                 return -1;
2871                         }
2872                 }
2873         }
2874
2875         return 0;
2876 }
2877
2878 static int
2879 worker_loopback_producer_fn(void *arg)
2880 {
2881         struct test *t = arg;
2882         uint8_t port = t->port[0];
2883         uint64_t count = 0;
2884
2885         printf("%d: \tProducer function started\n", __LINE__);
2886         while (count < NUM_PACKETS) {
2887                 struct rte_mbuf *m = 0;
2888                 do {
2889                         m = rte_pktmbuf_alloc(t->mbuf_pool);
2890                 } while (m == NULL);
2891
2892                 m->udata64 = 0;
2893
2894                 struct rte_event ev = {
2895                                 .op = RTE_EVENT_OP_NEW,
2896                                 .queue_id = t->qid[0],
2897                                 .flow_id = (uintptr_t)m & 0xFFFF,
2898                                 .mbuf = m,
2899                 };
2900
2901                 if (rte_event_enqueue_burst(evdev, port, &ev, 1) != 1) {
2902                         while (rte_event_enqueue_burst(evdev, port, &ev, 1) !=
2903                                         1)
2904                                 rte_pause();
2905                 }
2906
2907                 count++;
2908         }
2909
2910         return 0;
2911 }
2912
2913 static int
2914 worker_loopback(struct test *t)
2915 {
2916         /* use a single producer core, and a worker core to see what happens
2917          * if the worker loops packets back multiple times
2918          */
2919         struct test_event_dev_stats stats;
2920         uint64_t print_cycles = 0, cycles = 0;
2921         uint64_t tx_pkts = 0;
2922         int err;
2923         int w_lcore, p_lcore;
2924
2925         if (init(t, 8, 2) < 0 ||
2926                         create_atomic_qids(t, 8) < 0) {
2927                 printf("%d: Error initializing device\n", __LINE__);
2928                 return -1;
2929         }
2930
2931         /* RX with low max events */
2932         static struct rte_event_port_conf conf = {
2933                         .dequeue_depth = 32,
2934                         .enqueue_depth = 64,
2935         };
2936         /* beware: this cannot be initialized in the static above as it would
2937          * only be initialized once - and this needs to be set for multiple runs
2938          */
2939         conf.new_event_threshold = 512;
2940
2941         if (rte_event_port_setup(evdev, 0, &conf) < 0) {
2942                 printf("Error setting up RX port\n");
2943                 return -1;
2944         }
2945         t->port[0] = 0;
2946         /* TX with higher max events */
2947         conf.new_event_threshold = 4096;
2948         if (rte_event_port_setup(evdev, 1, &conf) < 0) {
2949                 printf("Error setting up TX port\n");
2950                 return -1;
2951         }
2952         t->port[1] = 1;
2953
2954         /* CQ mapping to QID */
2955         err = rte_event_port_link(evdev, t->port[1], NULL, NULL, 0);
2956         if (err != 8) { /* should have mapped all queues*/
2957                 printf("%d: error mapping port 2 to all qids\n", __LINE__);
2958                 return -1;
2959         }
2960
2961         if (rte_event_dev_start(evdev) < 0) {
2962                 printf("%d: Error with start call\n", __LINE__);
2963                 return -1;
2964         }
2965
2966         p_lcore = rte_get_next_lcore(
2967                         /* start core */ -1,
2968                         /* skip master */ 1,
2969                         /* wrap */ 0);
2970         w_lcore = rte_get_next_lcore(p_lcore, 1, 0);
2971
2972         rte_eal_remote_launch(worker_loopback_producer_fn, t, p_lcore);
2973         rte_eal_remote_launch(worker_loopback_worker_fn, t, w_lcore);
2974
2975         print_cycles = cycles = rte_get_timer_cycles();
2976         while (rte_eal_get_lcore_state(p_lcore) != FINISHED ||
2977                         rte_eal_get_lcore_state(w_lcore) != FINISHED) {
2978
2979                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
2980
2981                 uint64_t new_cycles = rte_get_timer_cycles();
2982
2983                 if (new_cycles - print_cycles > rte_get_timer_hz()) {
2984                         test_event_dev_stats_get(evdev, &stats);
2985                         printf(
2986                                 "%d: \tSched Rx = %"PRIu64", Tx = %"PRIu64"\n",
2987                                 __LINE__, stats.rx_pkts, stats.tx_pkts);
2988
2989                         print_cycles = new_cycles;
2990                 }
2991                 if (new_cycles - cycles > rte_get_timer_hz() * 3) {
2992                         test_event_dev_stats_get(evdev, &stats);
2993                         if (stats.tx_pkts == tx_pkts) {
2994                                 rte_event_dev_dump(evdev, stdout);
2995                                 printf("Dumping xstats:\n");
2996                                 xstats_print();
2997                                 printf(
2998                                         "%d: No schedules for seconds, deadlock\n",
2999                                         __LINE__);
3000                                 return -1;
3001                         }
3002                         tx_pkts = stats.tx_pkts;
3003                         cycles = new_cycles;
3004                 }
3005         }
3006         rte_service_run_iter_on_app_lcore(t->service_id, 1);
3007         /* ensure all completions are flushed */
3008
3009         rte_eal_mp_wait_lcore();
3010
3011         cleanup(t);
3012         return 0;
3013 }
3014
3015 static struct rte_mempool *eventdev_func_mempool;
3016
3017 static int
3018 test_sw_eventdev(void)
3019 {
3020         struct test *t = malloc(sizeof(struct test));
3021         int ret;
3022
3023         /* manually initialize the op, older gcc's complain on static
3024          * initialization of struct elements that are a bitfield.
3025          */
3026         release_ev.op = RTE_EVENT_OP_RELEASE;
3027
3028         const char *eventdev_name = "event_sw0";
3029         evdev = rte_event_dev_get_dev_id(eventdev_name);
3030         if (evdev < 0) {
3031                 printf("%d: Eventdev %s not found - creating.\n",
3032                                 __LINE__, eventdev_name);
3033                 if (rte_vdev_init(eventdev_name, NULL) < 0) {
3034                         printf("Error creating eventdev\n");
3035                         return -1;
3036                 }
3037                 evdev = rte_event_dev_get_dev_id(eventdev_name);
3038                 if (evdev < 0) {
3039                         printf("Error finding newly created eventdev\n");
3040                         return -1;
3041                 }
3042         }
3043
3044         if (rte_event_dev_service_id_get(evdev, &t->service_id) < 0) {
3045                 printf("Failed to get service ID for software event dev\n");
3046                 return -1;
3047         }
3048
3049         rte_service_runstate_set(t->service_id, 1);
3050         rte_service_set_runstate_mapped_check(t->service_id, 0);
3051
3052         /* Only create mbuf pool once, reuse for each test run */
3053         if (!eventdev_func_mempool) {
3054                 eventdev_func_mempool = rte_pktmbuf_pool_create(
3055                                 "EVENTDEV_SW_SA_MBUF_POOL",
3056                                 (1<<12), /* 4k buffers */
3057                                 32 /*MBUF_CACHE_SIZE*/,
3058                                 0,
3059                                 512, /* use very small mbufs */
3060                                 rte_socket_id());
3061                 if (!eventdev_func_mempool) {
3062                         printf("ERROR creating mempool\n");
3063                         return -1;
3064                 }
3065         }
3066         t->mbuf_pool = eventdev_func_mempool;
3067         printf("*** Running Single Directed Packet test...\n");
3068         ret = test_single_directed_packet(t);
3069         if (ret != 0) {
3070                 printf("ERROR - Single Directed Packet test FAILED.\n");
3071                 return ret;
3072         }
3073         printf("*** Running Directed Forward Credit test...\n");
3074         ret = test_directed_forward_credits(t);
3075         if (ret != 0) {
3076                 printf("ERROR - Directed Forward Credit test FAILED.\n");
3077                 return ret;
3078         }
3079         printf("*** Running Single Load Balanced Packet test...\n");
3080         ret = single_packet(t);
3081         if (ret != 0) {
3082                 printf("ERROR - Single Packet test FAILED.\n");
3083                 return ret;
3084         }
3085         printf("*** Running Unordered Basic test...\n");
3086         ret = unordered_basic(t);
3087         if (ret != 0) {
3088                 printf("ERROR -  Unordered Basic test FAILED.\n");
3089                 return ret;
3090         }
3091         printf("*** Running Ordered Basic test...\n");
3092         ret = ordered_basic(t);
3093         if (ret != 0) {
3094                 printf("ERROR -  Ordered Basic test FAILED.\n");
3095                 return ret;
3096         }
3097         printf("*** Running Burst Packets test...\n");
3098         ret = burst_packets(t);
3099         if (ret != 0) {
3100                 printf("ERROR - Burst Packets test FAILED.\n");
3101                 return ret;
3102         }
3103         printf("*** Running Load Balancing test...\n");
3104         ret = load_balancing(t);
3105         if (ret != 0) {
3106                 printf("ERROR - Load Balancing test FAILED.\n");
3107                 return ret;
3108         }
3109         printf("*** Running Prioritized Directed test...\n");
3110         ret = test_priority_directed(t);
3111         if (ret != 0) {
3112                 printf("ERROR - Prioritized Directed test FAILED.\n");
3113                 return ret;
3114         }
3115         printf("*** Running Prioritized Atomic test...\n");
3116         ret = test_priority_atomic(t);
3117         if (ret != 0) {
3118                 printf("ERROR - Prioritized Atomic test FAILED.\n");
3119                 return ret;
3120         }
3121
3122         printf("*** Running Prioritized Ordered test...\n");
3123         ret = test_priority_ordered(t);
3124         if (ret != 0) {
3125                 printf("ERROR - Prioritized Ordered test FAILED.\n");
3126                 return ret;
3127         }
3128         printf("*** Running Prioritized Unordered test...\n");
3129         ret = test_priority_unordered(t);
3130         if (ret != 0) {
3131                 printf("ERROR - Prioritized Unordered test FAILED.\n");
3132                 return ret;
3133         }
3134         printf("*** Running Invalid QID test...\n");
3135         ret = invalid_qid(t);
3136         if (ret != 0) {
3137                 printf("ERROR - Invalid QID test FAILED.\n");
3138                 return ret;
3139         }
3140         printf("*** Running Load Balancing History test...\n");
3141         ret = load_balancing_history(t);
3142         if (ret != 0) {
3143                 printf("ERROR - Load Balancing History test FAILED.\n");
3144                 return ret;
3145         }
3146         printf("*** Running Inflight Count test...\n");
3147         ret = inflight_counts(t);
3148         if (ret != 0) {
3149                 printf("ERROR - Inflight Count test FAILED.\n");
3150                 return ret;
3151         }
3152         printf("*** Running Abuse Inflights test...\n");
3153         ret = abuse_inflights(t);
3154         if (ret != 0) {
3155                 printf("ERROR - Abuse Inflights test FAILED.\n");
3156                 return ret;
3157         }
3158         printf("*** Running XStats test...\n");
3159         ret = xstats_tests(t);
3160         if (ret != 0) {
3161                 printf("ERROR - XStats test FAILED.\n");
3162                 return ret;
3163         }
3164         printf("*** Running XStats ID Reset test...\n");
3165         ret = xstats_id_reset_tests(t);
3166         if (ret != 0) {
3167                 printf("ERROR - XStats ID Reset test FAILED.\n");
3168                 return ret;
3169         }
3170         printf("*** Running XStats Brute Force test...\n");
3171         ret = xstats_brute_force(t);
3172         if (ret != 0) {
3173                 printf("ERROR - XStats Brute Force test FAILED.\n");
3174                 return ret;
3175         }
3176         printf("*** Running XStats ID Abuse test...\n");
3177         ret = xstats_id_abuse_tests(t);
3178         if (ret != 0) {
3179                 printf("ERROR - XStats ID Abuse test FAILED.\n");
3180                 return ret;
3181         }
3182         printf("*** Running QID Priority test...\n");
3183         ret = qid_priorities(t);
3184         if (ret != 0) {
3185                 printf("ERROR - QID Priority test FAILED.\n");
3186                 return ret;
3187         }
3188         printf("*** Running Ordered Reconfigure test...\n");
3189         ret = ordered_reconfigure(t);
3190         if (ret != 0) {
3191                 printf("ERROR - Ordered Reconfigure test FAILED.\n");
3192                 return ret;
3193         }
3194         printf("*** Running Port LB Single Reconfig test...\n");
3195         ret = port_single_lb_reconfig(t);
3196         if (ret != 0) {
3197                 printf("ERROR - Port LB Single Reconfig test FAILED.\n");
3198                 return ret;
3199         }
3200         printf("*** Running Port Reconfig Credits test...\n");
3201         ret = port_reconfig_credits(t);
3202         if (ret != 0) {
3203                 printf("ERROR - Port Reconfig Credits Reset test FAILED.\n");
3204                 return ret;
3205         }
3206         printf("*** Running Head-of-line-blocking test...\n");
3207         ret = holb(t);
3208         if (ret != 0) {
3209                 printf("ERROR - Head-of-line-blocking test FAILED.\n");
3210                 return ret;
3211         }
3212         if (rte_lcore_count() >= 3) {
3213                 printf("*** Running Worker loopback test...\n");
3214                 ret = worker_loopback(t);
3215                 if (ret != 0) {
3216                         printf("ERROR - Worker loopback test FAILED.\n");
3217                         return ret;
3218                 }
3219         } else {
3220                 printf("### Not enough cores for worker loopback test.\n");
3221                 printf("### Need at least 3 cores for test.\n");
3222         }
3223         /*
3224          * Free test instance, leaving mempool initialized, and a pointer to it
3225          * in static eventdev_func_mempool, as it is re-used on re-runs
3226          */
3227         free(t);
3228
3229         return 0;
3230 }
3231
3232 REGISTER_TEST_COMMAND(eventdev_sw_autotest, test_sw_eventdev);