drivers/event/sw/sw_evdev_selftest.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <string.h>
   7 #include <stdint.h>
   8 #include <errno.h>
   9 #include <unistd.h>
  10 #include <sys/queue.h>
  11
  12 #include <rte_memory.h>
  13 #include <rte_launch.h>
  14 #include <rte_eal.h>
  15 #include <rte_per_lcore.h>
  16 #include <rte_lcore.h>
  17 #include <rte_debug.h>
  18 #include <rte_ethdev.h>
  19 #include <rte_cycles.h>
  20 #include <rte_eventdev.h>
  21 #include <rte_pause.h>
  22 #include <rte_service.h>
  23 #include <rte_service_component.h>
  24 #include <rte_bus_vdev.h>
  25
  26 #include "sw_evdev.h"
  27
  28 #define MAX_PORTS 16
  29 #define MAX_QIDS 16
  30 #define NUM_PACKETS (1<<18)
  31 #define DEQUEUE_DEPTH 128
  32
  33 static int evdev;
  34
  35 struct test {
  36         struct rte_mempool *mbuf_pool;
  37         uint8_t port[MAX_PORTS];
  38         uint8_t qid[MAX_QIDS];
  39         int nb_qids;
  40         uint32_t service_id;
  41 };
  42
  43 static struct rte_event release_ev;
  44
  45 static inline struct rte_mbuf *
  46 rte_gen_arp(int portid, struct rte_mempool *mp)
  47 {
  48         /*
  49          * len = 14 + 46
  50          * ARP, Request who-has 10.0.0.1 tell 10.0.0.2, length 46
  51          */
  52         static const uint8_t arp_request[] = {
  53                 /*0x0000:*/ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xec, 0xa8,
  54                 0x6b, 0xfd, 0x02, 0x29, 0x08, 0x06, 0x00, 0x01,
  55                 /*0x0010:*/ 0x08, 0x00, 0x06, 0x04, 0x00, 0x01, 0xec, 0xa8,
  56                 0x6b, 0xfd, 0x02, 0x29, 0x0a, 0x00, 0x00, 0x01,
  57                 /*0x0020:*/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00,
  58                 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  59                 /*0x0030:*/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  60                 0x00, 0x00, 0x00, 0x00
  61         };
  62         struct rte_mbuf *m;
  63         int pkt_len = sizeof(arp_request) - 1;
  64
  65         m = rte_pktmbuf_alloc(mp);
  66         if (!m)
  67                 return 0;
  68
  69         memcpy((void *)((uintptr_t)m->buf_addr + m->data_off),
  70                 arp_request, pkt_len);
  71         rte_pktmbuf_pkt_len(m) = pkt_len;
  72         rte_pktmbuf_data_len(m) = pkt_len;
  73
  74         RTE_SET_USED(portid);
  75
  76         return m;
  77 }
  78
  79 static void
  80 xstats_print(void)
  81 {
  82         const uint32_t XSTATS_MAX = 1024;
  83         uint32_t i;
  84         uint32_t ids[XSTATS_MAX];
  85         uint64_t values[XSTATS_MAX];
  86         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
  87
  88         for (i = 0; i < XSTATS_MAX; i++)
  89                 ids[i] = i;
  90
  91         /* Device names / values */
  92         int ret = rte_event_dev_xstats_names_get(evdev,
  93                                         RTE_EVENT_DEV_XSTATS_DEVICE, 0,
  94                                         xstats_names, ids, XSTATS_MAX);
  95         if (ret < 0) {
  96                 printf("%d: xstats names get() returned error\n",
  97                         __LINE__);
  98                 return;
  99         }
 100         ret = rte_event_dev_xstats_get(evdev,
 101                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 102                                         0, ids, values, ret);
 103         if (ret > (signed int)XSTATS_MAX)
 104                 printf("%s %d: more xstats available than space\n",
 105                                 __func__, __LINE__);
 106         for (i = 0; (signed int)i < ret; i++) {
 107                 printf("%d : %s : %"PRIu64"\n",
 108                                 i, xstats_names[i].name, values[i]);
 109         }
 110
 111         /* Port names / values */
 112         ret = rte_event_dev_xstats_names_get(evdev,
 113                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
 114                                         xstats_names, ids, XSTATS_MAX);
 115         ret = rte_event_dev_xstats_get(evdev,
 116                                         RTE_EVENT_DEV_XSTATS_PORT, 1,
 117                                         ids, values, ret);
 118         if (ret > (signed int)XSTATS_MAX)
 119                 printf("%s %d: more xstats available than space\n",
 120                                 __func__, __LINE__);
 121         for (i = 0; (signed int)i < ret; i++) {
 122                 printf("%d : %s : %"PRIu64"\n",
 123                                 i, xstats_names[i].name, values[i]);
 124         }
 125
 126         /* Queue names / values */
 127         ret = rte_event_dev_xstats_names_get(evdev,
 128                                         RTE_EVENT_DEV_XSTATS_QUEUE, 0,
 129                                         xstats_names, ids, XSTATS_MAX);
 130         ret = rte_event_dev_xstats_get(evdev,
 131                                         RTE_EVENT_DEV_XSTATS_QUEUE,
 132                                         1, ids, values, ret);
 133         if (ret > (signed int)XSTATS_MAX)
 134                 printf("%s %d: more xstats available than space\n",
 135                                 __func__, __LINE__);
 136         for (i = 0; (signed int)i < ret; i++) {
 137                 printf("%d : %s : %"PRIu64"\n",
 138                                 i, xstats_names[i].name, values[i]);
 139         }
 140 }
 141
 142 /* initialization and config */
 143 static inline int
 144 init(struct test *t, int nb_queues, int nb_ports)
 145 {
 146         struct rte_event_dev_config config = {
 147                         .nb_event_queues = nb_queues,
 148                         .nb_event_ports = nb_ports,
 149                         .nb_event_queue_flows = 1024,
 150                         .nb_events_limit = 4096,
 151                         .nb_event_port_dequeue_depth = DEQUEUE_DEPTH,
 152                         .nb_event_port_enqueue_depth = 128,
 153         };
 154         int ret;
 155
 156         void *temp = t->mbuf_pool; /* save and restore mbuf pool */
 157
 158         memset(t, 0, sizeof(*t));
 159         t->mbuf_pool = temp;
 160
 161         ret = rte_event_dev_configure(evdev, &config);
 162         if (ret < 0)
 163                 printf("%d: Error configuring device\n", __LINE__);
 164         return ret;
 165 };
 166
 167 static inline int
 168 create_ports(struct test *t, int num_ports)
 169 {
 170         int i;
 171         static const struct rte_event_port_conf conf = {
 172                         .new_event_threshold = 1024,
 173                         .dequeue_depth = 32,
 174                         .enqueue_depth = 64,
 175         };
 176         if (num_ports > MAX_PORTS)
 177                 return -1;
 178
 179         for (i = 0; i < num_ports; i++) {
 180                 if (rte_event_port_setup(evdev, i, &conf) < 0) {
 181                         printf("Error setting up port %d\n", i);
 182                         return -1;
 183                 }
 184                 t->port[i] = i;
 185         }
 186
 187         return 0;
 188 }
 189
 190 static inline int
 191 create_lb_qids(struct test *t, int num_qids, uint32_t flags)
 192 {
 193         int i;
 194
 195         /* Q creation */
 196         const struct rte_event_queue_conf conf = {
 197                         .schedule_type = flags,
 198                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 199                         .nb_atomic_flows = 1024,
 200                         .nb_atomic_order_sequences = 1024,
 201         };
 202
 203         for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
 204                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
 205                         printf("%d: error creating qid %d\n", __LINE__, i);
 206                         return -1;
 207                 }
 208                 t->qid[i] = i;
 209         }
 210         t->nb_qids += num_qids;
 211         if (t->nb_qids > MAX_QIDS)
 212                 return -1;
 213
 214         return 0;
 215 }
 216
 217 static inline int
 218 create_atomic_qids(struct test *t, int num_qids)
 219 {
 220         return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
 221 }
 222
 223 static inline int
 224 create_ordered_qids(struct test *t, int num_qids)
 225 {
 226         return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ORDERED);
 227 }
 228
 229
 230 static inline int
 231 create_unordered_qids(struct test *t, int num_qids)
 232 {
 233         return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_PARALLEL);
 234 }
 235
 236 static inline int
 237 create_directed_qids(struct test *t, int num_qids, const uint8_t ports[])
 238 {
 239         int i;
 240
 241         /* Q creation */
 242         static const struct rte_event_queue_conf conf = {
 243                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 244                         .event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK,
 245         };
 246
 247         for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
 248                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
 249                         printf("%d: error creating qid %d\n", __LINE__, i);
 250                         return -1;
 251                 }
 252                 t->qid[i] = i;
 253
 254                 if (rte_event_port_link(evdev, ports[i - t->nb_qids],
 255                                 &t->qid[i], NULL, 1) != 1) {
 256                         printf("%d: error creating link for qid %d\n",
 257                                         __LINE__, i);
 258                         return -1;
 259                 }
 260         }
 261         t->nb_qids += num_qids;
 262         if (t->nb_qids > MAX_QIDS)
 263                 return -1;
 264
 265         return 0;
 266 }
 267
 268 /* destruction */
 269 static inline int
 270 cleanup(struct test *t __rte_unused)
 271 {
 272         rte_event_dev_stop(evdev);
 273         rte_event_dev_close(evdev);
 274         return 0;
 275 };
 276
 277 struct test_event_dev_stats {
 278         uint64_t rx_pkts;       /**< Total packets received */
 279         uint64_t rx_dropped;    /**< Total packets dropped (Eg Invalid QID) */
 280         uint64_t tx_pkts;       /**< Total packets transmitted */
 281
 282         /** Packets received on this port */
 283         uint64_t port_rx_pkts[MAX_PORTS];
 284         /** Packets dropped on this port */
 285         uint64_t port_rx_dropped[MAX_PORTS];
 286         /** Packets inflight on this port */
 287         uint64_t port_inflight[MAX_PORTS];
 288         /** Packets transmitted on this port */
 289         uint64_t port_tx_pkts[MAX_PORTS];
 290         /** Packets received on this qid */
 291         uint64_t qid_rx_pkts[MAX_QIDS];
 292         /** Packets dropped on this qid */
 293         uint64_t qid_rx_dropped[MAX_QIDS];
 294         /** Packets transmitted on this qid */
 295         uint64_t qid_tx_pkts[MAX_QIDS];
 296 };
 297
 298 static inline int
 299 test_event_dev_stats_get(int dev_id, struct test_event_dev_stats *stats)
 300 {
 301         static uint32_t i;
 302         static uint32_t total_ids[3]; /* rx, tx and drop */
 303         static uint32_t port_rx_pkts_ids[MAX_PORTS];
 304         static uint32_t port_rx_dropped_ids[MAX_PORTS];
 305         static uint32_t port_inflight_ids[MAX_PORTS];
 306         static uint32_t port_tx_pkts_ids[MAX_PORTS];
 307         static uint32_t qid_rx_pkts_ids[MAX_QIDS];
 308         static uint32_t qid_rx_dropped_ids[MAX_QIDS];
 309         static uint32_t qid_tx_pkts_ids[MAX_QIDS];
 310
 311
 312         stats->rx_pkts = rte_event_dev_xstats_by_name_get(dev_id,
 313                         "dev_rx", &total_ids[0]);
 314         stats->rx_dropped = rte_event_dev_xstats_by_name_get(dev_id,
 315                         "dev_drop", &total_ids[1]);
 316         stats->tx_pkts = rte_event_dev_xstats_by_name_get(dev_id,
 317                         "dev_tx", &total_ids[2]);
 318         for (i = 0; i < MAX_PORTS; i++) {
 319                 char name[32];
 320                 snprintf(name, sizeof(name), "port_%u_rx", i);
 321                 stats->port_rx_pkts[i] = rte_event_dev_xstats_by_name_get(
 322                                 dev_id, name, &port_rx_pkts_ids[i]);
 323                 snprintf(name, sizeof(name), "port_%u_drop", i);
 324                 stats->port_rx_dropped[i] = rte_event_dev_xstats_by_name_get(
 325                                 dev_id, name, &port_rx_dropped_ids[i]);
 326                 snprintf(name, sizeof(name), "port_%u_inflight", i);
 327                 stats->port_inflight[i] = rte_event_dev_xstats_by_name_get(
 328                                 dev_id, name, &port_inflight_ids[i]);
 329                 snprintf(name, sizeof(name), "port_%u_tx", i);
 330                 stats->port_tx_pkts[i] = rte_event_dev_xstats_by_name_get(
 331                                 dev_id, name, &port_tx_pkts_ids[i]);
 332         }
 333         for (i = 0; i < MAX_QIDS; i++) {
 334                 char name[32];
 335                 snprintf(name, sizeof(name), "qid_%u_rx", i);
 336                 stats->qid_rx_pkts[i] = rte_event_dev_xstats_by_name_get(
 337                                 dev_id, name, &qid_rx_pkts_ids[i]);
 338                 snprintf(name, sizeof(name), "qid_%u_drop", i);
 339                 stats->qid_rx_dropped[i] = rte_event_dev_xstats_by_name_get(
 340                                 dev_id, name, &qid_rx_dropped_ids[i]);
 341                 snprintf(name, sizeof(name), "qid_%u_tx", i);
 342                 stats->qid_tx_pkts[i] = rte_event_dev_xstats_by_name_get(
 343                                 dev_id, name, &qid_tx_pkts_ids[i]);
 344         }
 345
 346         return 0;
 347 }
 348
 349 /* run_prio_packet_test
 350  * This performs a basic packet priority check on the test instance passed in.
 351  * It is factored out of the main priority tests as the same tests must be
 352  * performed to ensure prioritization of each type of QID.
 353  *
 354  * Requirements:
 355  *  - An initialized test structure, including mempool
 356  *  - t->port[0] is initialized for both Enq / Deq of packets to the QID
 357  *  - t->qid[0] is the QID to be tested
 358  *  - if LB QID, the CQ must be mapped to the QID.
 359  */
 360 static int
 361 run_prio_packet_test(struct test *t)
 362 {
 363         int err;
 364         const uint32_t MAGIC_SEQN[] = {4711, 1234};
 365         const uint32_t PRIORITY[] = {
 366                 RTE_EVENT_DEV_PRIORITY_NORMAL,
 367                 RTE_EVENT_DEV_PRIORITY_HIGHEST
 368         };
 369         unsigned int i;
 370         for (i = 0; i < RTE_DIM(MAGIC_SEQN); i++) {
 371                 /* generate pkt and enqueue */
 372                 struct rte_event ev;
 373                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
 374                 if (!arp) {
 375                         printf("%d: gen of pkt failed\n", __LINE__);
 376                         return -1;
 377                 }
 378                 arp->seqn = MAGIC_SEQN[i];
 379
 380                 ev = (struct rte_event){
 381                         .priority = PRIORITY[i],
 382                         .op = RTE_EVENT_OP_NEW,
 383                         .queue_id = t->qid[0],
 384                         .mbuf = arp
 385                 };
 386                 err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
 387                 if (err != 1) {
 388                         printf("%d: error failed to enqueue\n", __LINE__);
 389                         return -1;
 390                 }
 391         }
 392
 393         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 394
 395         struct test_event_dev_stats stats;
 396         err = test_event_dev_stats_get(evdev, &stats);
 397         if (err) {
 398                 printf("%d: error failed to get stats\n", __LINE__);
 399                 return -1;
 400         }
 401
 402         if (stats.port_rx_pkts[t->port[0]] != 2) {
 403                 printf("%d: error stats incorrect for directed port\n",
 404                                 __LINE__);
 405                 rte_event_dev_dump(evdev, stdout);
 406                 return -1;
 407         }
 408
 409         struct rte_event ev, ev2;
 410         uint32_t deq_pkts;
 411         deq_pkts = rte_event_dequeue_burst(evdev, t->port[0], &ev, 1, 0);
 412         if (deq_pkts != 1) {
 413                 printf("%d: error failed to deq\n", __LINE__);
 414                 rte_event_dev_dump(evdev, stdout);
 415                 return -1;
 416         }
 417         if (ev.mbuf->seqn != MAGIC_SEQN[1]) {
 418                 printf("%d: first packet out not highest priority\n",
 419                                 __LINE__);
 420                 rte_event_dev_dump(evdev, stdout);
 421                 return -1;
 422         }
 423         rte_pktmbuf_free(ev.mbuf);
 424
 425         deq_pkts = rte_event_dequeue_burst(evdev, t->port[0], &ev2, 1, 0);
 426         if (deq_pkts != 1) {
 427                 printf("%d: error failed to deq\n", __LINE__);
 428                 rte_event_dev_dump(evdev, stdout);
 429                 return -1;
 430         }
 431         if (ev2.mbuf->seqn != MAGIC_SEQN[0]) {
 432                 printf("%d: second packet out not lower priority\n",
 433                                 __LINE__);
 434                 rte_event_dev_dump(evdev, stdout);
 435                 return -1;
 436         }
 437         rte_pktmbuf_free(ev2.mbuf);
 438
 439         cleanup(t);
 440         return 0;
 441 }
 442
 443 static int
 444 test_single_directed_packet(struct test *t)
 445 {
 446         const int rx_enq = 0;
 447         const int wrk_enq = 2;
 448         int err;
 449
 450         /* Create instance with 3 directed QIDs going to 3 ports */
 451         if (init(t, 3, 3) < 0 ||
 452                         create_ports(t, 3) < 0 ||
 453                         create_directed_qids(t, 3, t->port) < 0)
 454                 return -1;
 455
 456         if (rte_event_dev_start(evdev) < 0) {
 457                 printf("%d: Error with start call\n", __LINE__);
 458                 return -1;
 459         }
 460
 461         /************** FORWARD ****************/
 462         struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
 463         struct rte_event ev = {
 464                         .op = RTE_EVENT_OP_NEW,
 465                         .queue_id = wrk_enq,
 466                         .mbuf = arp,
 467         };
 468
 469         if (!arp) {
 470                 printf("%d: gen of pkt failed\n", __LINE__);
 471                 return -1;
 472         }
 473
 474         const uint32_t MAGIC_SEQN = 4711;
 475         arp->seqn = MAGIC_SEQN;
 476
 477         /* generate pkt and enqueue */
 478         err = rte_event_enqueue_burst(evdev, rx_enq, &ev, 1);
 479         if (err != 1) {
 480                 printf("%d: error failed to enqueue\n", __LINE__);
 481                 return -1;
 482         }
 483
 484         /* Run schedule() as dir packets may need to be re-ordered */
 485         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 486
 487         struct test_event_dev_stats stats;
 488         err = test_event_dev_stats_get(evdev, &stats);
 489         if (err) {
 490                 printf("%d: error failed to get stats\n", __LINE__);
 491                 return -1;
 492         }
 493
 494         if (stats.port_rx_pkts[rx_enq] != 1) {
 495                 printf("%d: error stats incorrect for directed port\n",
 496                                 __LINE__);
 497                 return -1;
 498         }
 499
 500         uint32_t deq_pkts;
 501         deq_pkts = rte_event_dequeue_burst(evdev, wrk_enq, &ev, 1, 0);
 502         if (deq_pkts != 1) {
 503                 printf("%d: error failed to deq\n", __LINE__);
 504                 return -1;
 505         }
 506
 507         err = test_event_dev_stats_get(evdev, &stats);
 508         if (stats.port_rx_pkts[wrk_enq] != 0 &&
 509                         stats.port_rx_pkts[wrk_enq] != 1) {
 510                 printf("%d: error directed stats post-dequeue\n", __LINE__);
 511                 return -1;
 512         }
 513
 514         if (ev.mbuf->seqn != MAGIC_SEQN) {
 515                 printf("%d: error magic sequence number not dequeued\n",
 516                                 __LINE__);
 517                 return -1;
 518         }
 519
 520         rte_pktmbuf_free(ev.mbuf);
 521         cleanup(t);
 522         return 0;
 523 }
 524
 525 static int
 526 test_directed_forward_credits(struct test *t)
 527 {
 528         uint32_t i;
 529         int32_t err;
 530
 531         if (init(t, 1, 1) < 0 ||
 532                         create_ports(t, 1) < 0 ||
 533                         create_directed_qids(t, 1, t->port) < 0)
 534                 return -1;
 535
 536         if (rte_event_dev_start(evdev) < 0) {
 537                 printf("%d: Error with start call\n", __LINE__);
 538                 return -1;
 539         }
 540
 541         struct rte_event ev = {
 542                         .op = RTE_EVENT_OP_NEW,
 543                         .queue_id = 0,
 544         };
 545
 546         for (i = 0; i < 1000; i++) {
 547                 err = rte_event_enqueue_burst(evdev, 0, &ev, 1);
 548                 if (err != 1) {
 549                         printf("%d: error failed to enqueue\n", __LINE__);
 550                         return -1;
 551                 }
 552                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
 553
 554                 uint32_t deq_pkts;
 555                 deq_pkts = rte_event_dequeue_burst(evdev, 0, &ev, 1, 0);
 556                 if (deq_pkts != 1) {
 557                         printf("%d: error failed to deq\n", __LINE__);
 558                         return -1;
 559                 }
 560
 561                 /* re-write event to be a forward, and continue looping it */
 562                 ev.op = RTE_EVENT_OP_FORWARD;
 563         }
 564
 565         cleanup(t);
 566         return 0;
 567 }
 568
 569
 570 static int
 571 test_priority_directed(struct test *t)
 572 {
 573         if (init(t, 1, 1) < 0 ||
 574                         create_ports(t, 1) < 0 ||
 575                         create_directed_qids(t, 1, t->port) < 0) {
 576                 printf("%d: Error initializing device\n", __LINE__);
 577                 return -1;
 578         }
 579
 580         if (rte_event_dev_start(evdev) < 0) {
 581                 printf("%d: Error with start call\n", __LINE__);
 582                 return -1;
 583         }
 584
 585         return run_prio_packet_test(t);
 586 }
 587
 588 static int
 589 test_priority_atomic(struct test *t)
 590 {
 591         if (init(t, 1, 1) < 0 ||
 592                         create_ports(t, 1) < 0 ||
 593                         create_atomic_qids(t, 1) < 0) {
 594                 printf("%d: Error initializing device\n", __LINE__);
 595                 return -1;
 596         }
 597
 598         /* map the QID */
 599         if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) {
 600                 printf("%d: error mapping qid to port\n", __LINE__);
 601                 return -1;
 602         }
 603         if (rte_event_dev_start(evdev) < 0) {
 604                 printf("%d: Error with start call\n", __LINE__);
 605                 return -1;
 606         }
 607
 608         return run_prio_packet_test(t);
 609 }
 610
 611 static int
 612 test_priority_ordered(struct test *t)
 613 {
 614         if (init(t, 1, 1) < 0 ||
 615                         create_ports(t, 1) < 0 ||
 616                         create_ordered_qids(t, 1) < 0) {
 617                 printf("%d: Error initializing device\n", __LINE__);
 618                 return -1;
 619         }
 620
 621         /* map the QID */
 622         if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) {
 623                 printf("%d: error mapping qid to port\n", __LINE__);
 624                 return -1;
 625         }
 626         if (rte_event_dev_start(evdev) < 0) {
 627                 printf("%d: Error with start call\n", __LINE__);
 628                 return -1;
 629         }
 630
 631         return run_prio_packet_test(t);
 632 }
 633
 634 static int
 635 test_priority_unordered(struct test *t)
 636 {
 637         if (init(t, 1, 1) < 0 ||
 638                         create_ports(t, 1) < 0 ||
 639                         create_unordered_qids(t, 1) < 0) {
 640                 printf("%d: Error initializing device\n", __LINE__);
 641                 return -1;
 642         }
 643
 644         /* map the QID */
 645         if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) {
 646                 printf("%d: error mapping qid to port\n", __LINE__);
 647                 return -1;
 648         }
 649         if (rte_event_dev_start(evdev) < 0) {
 650                 printf("%d: Error with start call\n", __LINE__);
 651                 return -1;
 652         }
 653
 654         return run_prio_packet_test(t);
 655 }
 656
 657 static int
 658 burst_packets(struct test *t)
 659 {
 660         /************** CONFIG ****************/
 661         uint32_t i;
 662         int err;
 663         int ret;
 664
 665         /* Create instance with 2 ports and 2 queues */
 666         if (init(t, 2, 2) < 0 ||
 667                         create_ports(t, 2) < 0 ||
 668                         create_atomic_qids(t, 2) < 0) {
 669                 printf("%d: Error initializing device\n", __LINE__);
 670                 return -1;
 671         }
 672
 673         /* CQ mapping to QID */
 674         ret = rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1);
 675         if (ret != 1) {
 676                 printf("%d: error mapping lb qid0\n", __LINE__);
 677                 return -1;
 678         }
 679         ret = rte_event_port_link(evdev, t->port[1], &t->qid[1], NULL, 1);
 680         if (ret != 1) {
 681                 printf("%d: error mapping lb qid1\n", __LINE__);
 682                 return -1;
 683         }
 684
 685         if (rte_event_dev_start(evdev) < 0) {
 686                 printf("%d: Error with start call\n", __LINE__);
 687                 return -1;
 688         }
 689
 690         /************** FORWARD ****************/
 691         const uint32_t rx_port = 0;
 692         const uint32_t NUM_PKTS = 2;
 693
 694         for (i = 0; i < NUM_PKTS; i++) {
 695                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
 696                 if (!arp) {
 697                         printf("%d: error generating pkt\n", __LINE__);
 698                         return -1;
 699                 }
 700
 701                 struct rte_event ev = {
 702                                 .op = RTE_EVENT_OP_NEW,
 703                                 .queue_id = i % 2,
 704                                 .flow_id = i % 3,
 705                                 .mbuf = arp,
 706                 };
 707                 /* generate pkt and enqueue */
 708                 err = rte_event_enqueue_burst(evdev, t->port[rx_port], &ev, 1);
 709                 if (err != 1) {
 710                         printf("%d: Failed to enqueue\n", __LINE__);
 711                         return -1;
 712                 }
 713         }
 714         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 715
 716         /* Check stats for all NUM_PKTS arrived to sched core */
 717         struct test_event_dev_stats stats;
 718
 719         err = test_event_dev_stats_get(evdev, &stats);
 720         if (err) {
 721                 printf("%d: failed to get stats\n", __LINE__);
 722                 return -1;
 723         }
 724         if (stats.rx_pkts != NUM_PKTS || stats.tx_pkts != NUM_PKTS) {
 725                 printf("%d: Sched core didn't receive all %d pkts\n",
 726                                 __LINE__, NUM_PKTS);
 727                 rte_event_dev_dump(evdev, stdout);
 728                 return -1;
 729         }
 730
 731         uint32_t deq_pkts;
 732         int p;
 733
 734         deq_pkts = 0;
 735         /******** DEQ QID 1 *******/
 736         do {
 737                 struct rte_event ev;
 738                 p = rte_event_dequeue_burst(evdev, t->port[0], &ev, 1, 0);
 739                 deq_pkts += p;
 740                 rte_pktmbuf_free(ev.mbuf);
 741         } while (p);
 742
 743         if (deq_pkts != NUM_PKTS/2) {
 744                 printf("%d: Half of NUM_PKTS didn't arrive at port 1\n",
 745                                 __LINE__);
 746                 return -1;
 747         }
 748
 749         /******** DEQ QID 2 *******/
 750         deq_pkts = 0;
 751         do {
 752                 struct rte_event ev;
 753                 p = rte_event_dequeue_burst(evdev, t->port[1], &ev, 1, 0);
 754                 deq_pkts += p;
 755                 rte_pktmbuf_free(ev.mbuf);
 756         } while (p);
 757         if (deq_pkts != NUM_PKTS/2) {
 758                 printf("%d: Half of NUM_PKTS didn't arrive at port 2\n",
 759                                 __LINE__);
 760                 return -1;
 761         }
 762
 763         cleanup(t);
 764         return 0;
 765 }
 766
 767 static int
 768 abuse_inflights(struct test *t)
 769 {
 770         const int rx_enq = 0;
 771         const int wrk_enq = 2;
 772         int err;
 773
 774         /* Create instance with 4 ports */
 775         if (init(t, 1, 4) < 0 ||
 776                         create_ports(t, 4) < 0 ||
 777                         create_atomic_qids(t, 1) < 0) {
 778                 printf("%d: Error initializing device\n", __LINE__);
 779                 return -1;
 780         }
 781
 782         /* CQ mapping to QID */
 783         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
 784         if (err != 1) {
 785                 printf("%d: error mapping lb qid\n", __LINE__);
 786                 cleanup(t);
 787                 return -1;
 788         }
 789
 790         if (rte_event_dev_start(evdev) < 0) {
 791                 printf("%d: Error with start call\n", __LINE__);
 792                 return -1;
 793         }
 794
 795         /* Enqueue op only */
 796         err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &release_ev, 1);
 797         if (err != 1) {
 798                 printf("%d: Failed to enqueue\n", __LINE__);
 799                 return -1;
 800         }
 801
 802         /* schedule */
 803         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 804
 805         struct test_event_dev_stats stats;
 806
 807         err = test_event_dev_stats_get(evdev, &stats);
 808         if (err) {
 809                 printf("%d: failed to get stats\n", __LINE__);
 810                 return -1;
 811         }
 812
 813         if (stats.rx_pkts != 0 ||
 814                         stats.tx_pkts != 0 ||
 815                         stats.port_inflight[wrk_enq] != 0) {
 816                 printf("%d: Sched core didn't handle pkt as expected\n",
 817                                 __LINE__);
 818                 return -1;
 819         }
 820
 821         cleanup(t);
 822         return 0;
 823 }
 824
 825 static int
 826 xstats_tests(struct test *t)
 827 {
 828         const int wrk_enq = 2;
 829         int err;
 830
 831         /* Create instance with 4 ports */
 832         if (init(t, 1, 4) < 0 ||
 833                         create_ports(t, 4) < 0 ||
 834                         create_atomic_qids(t, 1) < 0) {
 835                 printf("%d: Error initializing device\n", __LINE__);
 836                 return -1;
 837         }
 838
 839         /* CQ mapping to QID */
 840         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
 841         if (err != 1) {
 842                 printf("%d: error mapping lb qid\n", __LINE__);
 843                 cleanup(t);
 844                 return -1;
 845         }
 846
 847         if (rte_event_dev_start(evdev) < 0) {
 848                 printf("%d: Error with start call\n", __LINE__);
 849                 return -1;
 850         }
 851
 852         const uint32_t XSTATS_MAX = 1024;
 853
 854         uint32_t i;
 855         uint32_t ids[XSTATS_MAX];
 856         uint64_t values[XSTATS_MAX];
 857         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
 858
 859         for (i = 0; i < XSTATS_MAX; i++)
 860                 ids[i] = i;
 861
 862         /* Device names / values */
 863         int ret = rte_event_dev_xstats_names_get(evdev,
 864                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 865                                         0, xstats_names, ids, XSTATS_MAX);
 866         if (ret != 6) {
 867                 printf("%d: expected 6 stats, got return %d\n", __LINE__, ret);
 868                 return -1;
 869         }
 870         ret = rte_event_dev_xstats_get(evdev,
 871                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 872                                         0, ids, values, ret);
 873         if (ret != 6) {
 874                 printf("%d: expected 6 stats, got return %d\n", __LINE__, ret);
 875                 return -1;
 876         }
 877
 878         /* Port names / values */
 879         ret = rte_event_dev_xstats_names_get(evdev,
 880                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
 881                                         xstats_names, ids, XSTATS_MAX);
 882         if (ret != 21) {
 883                 printf("%d: expected 21 stats, got return %d\n", __LINE__, ret);
 884                 return -1;
 885         }
 886         ret = rte_event_dev_xstats_get(evdev,
 887                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
 888                                         ids, values, ret);
 889         if (ret != 21) {
 890                 printf("%d: expected 21 stats, got return %d\n", __LINE__, ret);
 891                 return -1;
 892         }
 893
 894         /* Queue names / values */
 895         ret = rte_event_dev_xstats_names_get(evdev,
 896                                         RTE_EVENT_DEV_XSTATS_QUEUE,
 897                                         0, xstats_names, ids, XSTATS_MAX);
 898         if (ret != 16) {
 899                 printf("%d: expected 16 stats, got return %d\n", __LINE__, ret);
 900                 return -1;
 901         }
 902
 903         /* NEGATIVE TEST: with wrong queue passed, 0 stats should be returned */
 904         ret = rte_event_dev_xstats_get(evdev,
 905                                         RTE_EVENT_DEV_XSTATS_QUEUE,
 906                                         1, ids, values, ret);
 907         if (ret != -EINVAL) {
 908                 printf("%d: expected 0 stats, got return %d\n", __LINE__, ret);
 909                 return -1;
 910         }
 911
 912         ret = rte_event_dev_xstats_get(evdev,
 913                                         RTE_EVENT_DEV_XSTATS_QUEUE,
 914                                         0, ids, values, ret);
 915         if (ret != 16) {
 916                 printf("%d: expected 16 stats, got return %d\n", __LINE__, ret);
 917                 return -1;
 918         }
 919
 920         /* enqueue packets to check values */
 921         for (i = 0; i < 3; i++) {
 922                 struct rte_event ev;
 923                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
 924                 if (!arp) {
 925                         printf("%d: gen of pkt failed\n", __LINE__);
 926                         return -1;
 927                 }
 928                 ev.queue_id = t->qid[i];
 929                 ev.op = RTE_EVENT_OP_NEW;
 930                 ev.mbuf = arp;
 931                 ev.flow_id = 7;
 932                 arp->seqn = i;
 933
 934                 int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
 935                 if (err != 1) {
 936                         printf("%d: Failed to enqueue\n", __LINE__);
 937                         return -1;
 938                 }
 939         }
 940
 941         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 942
 943         /* Device names / values */
 944         int num_stats = rte_event_dev_xstats_names_get(evdev,
 945                                         RTE_EVENT_DEV_XSTATS_DEVICE, 0,
 946                                         xstats_names, ids, XSTATS_MAX);
 947         if (num_stats < 0)
 948                 goto fail;
 949         ret = rte_event_dev_xstats_get(evdev,
 950                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 951                                         0, ids, values, num_stats);
 952         static const uint64_t expected[] = {3, 3, 0, 1, 0, 0};
 953         for (i = 0; (signed int)i < ret; i++) {
 954                 if (expected[i] != values[i]) {
 955                         printf(
 956                                 "%d Error xstat %d (id %d) %s : %"PRIu64
 957                                 ", expect %"PRIu64"\n",
 958                                 __LINE__, i, ids[i], xstats_names[i].name,
 959                                 values[i], expected[i]);
 960                         goto fail;
 961                 }
 962         }
 963
 964         ret = rte_event_dev_xstats_reset(evdev, RTE_EVENT_DEV_XSTATS_DEVICE,
 965                                         0, NULL, 0);
 966
 967         /* ensure reset statistics are zero-ed */
 968         static const uint64_t expected_zero[] = {0, 0, 0, 0, 0, 0};
 969         ret = rte_event_dev_xstats_get(evdev,
 970                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 971                                         0, ids, values, num_stats);
 972         for (i = 0; (signed int)i < ret; i++) {
 973                 if (expected_zero[i] != values[i]) {
 974                         printf(
 975                                 "%d Error, xstat %d (id %d) %s : %"PRIu64
 976                                 ", expect %"PRIu64"\n",
 977                                 __LINE__, i, ids[i], xstats_names[i].name,
 978                                 values[i], expected_zero[i]);
 979                         goto fail;
 980                 }
 981         }
 982
 983         /* port reset checks */
 984         num_stats = rte_event_dev_xstats_names_get(evdev,
 985                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
 986                                         xstats_names, ids, XSTATS_MAX);
 987         if (num_stats < 0)
 988                 goto fail;
 989         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_PORT,
 990                                         0, ids, values, num_stats);
 991
 992         static const uint64_t port_expected[] = {
 993                 3 /* rx */,
 994                 0 /* tx */,
 995                 0 /* drop */,
 996                 0 /* inflights */,
 997                 0 /* avg pkt cycles */,
 998                 29 /* credits */,
 999                 0 /* rx ring used */,
1000                 4096 /* rx ring free */,
1001                 0 /* cq ring used */,
1002                 32 /* cq ring free */,
1003                 0 /* dequeue calls */,
1004                 /* 10 dequeue burst buckets */
1005                 0, 0, 0, 0, 0,
1006                 0, 0, 0, 0, 0,
1007         };
1008         if (ret != RTE_DIM(port_expected)) {
1009                 printf(
1010                         "%s %d: wrong number of port stats (%d), expected %zu\n",
1011                         __func__, __LINE__, ret, RTE_DIM(port_expected));
1012         }
1013
1014         for (i = 0; (signed int)i < ret; i++) {
1015                 if (port_expected[i] != values[i]) {
1016                         printf(
1017                                 "%s : %d: Error stat %s is %"PRIu64
1018                                 ", expected %"PRIu64"\n",
1019                                 __func__, __LINE__, xstats_names[i].name,
1020                                 values[i], port_expected[i]);
1021                         goto fail;
1022                 }
1023         }
1024
1025         ret = rte_event_dev_xstats_reset(evdev, RTE_EVENT_DEV_XSTATS_PORT,
1026                                         0, NULL, 0);
1027
1028         /* ensure reset statistics are zero-ed */
1029         static const uint64_t port_expected_zero[] = {
1030                 0 /* rx */,
1031                 0 /* tx */,
1032                 0 /* drop */,
1033                 0 /* inflights */,
1034                 0 /* avg pkt cycles */,
1035                 29 /* credits */,
1036                 0 /* rx ring used */,
1037                 4096 /* rx ring free */,
1038                 0 /* cq ring used */,
1039                 32 /* cq ring free */,
1040                 0 /* dequeue calls */,
1041                 /* 10 dequeue burst buckets */
1042                 0, 0, 0, 0, 0,
1043                 0, 0, 0, 0, 0,
1044         };
1045         ret = rte_event_dev_xstats_get(evdev,
1046                                         RTE_EVENT_DEV_XSTATS_PORT,
1047                                         0, ids, values, num_stats);
1048         for (i = 0; (signed int)i < ret; i++) {
1049                 if (port_expected_zero[i] != values[i]) {
1050                         printf(
1051                                 "%d, Error, xstat %d (id %d) %s : %"PRIu64
1052                                 ", expect %"PRIu64"\n",
1053                                 __LINE__, i, ids[i], xstats_names[i].name,
1054                                 values[i], port_expected_zero[i]);
1055                         goto fail;
1056                 }
1057         }
1058
1059         /* QUEUE STATS TESTS */
1060         num_stats = rte_event_dev_xstats_names_get(evdev,
1061                                                 RTE_EVENT_DEV_XSTATS_QUEUE, 0,
1062                                                 xstats_names, ids, XSTATS_MAX);
1063         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE,
1064                                         0, ids, values, num_stats);
1065         if (ret < 0) {
1066                 printf("xstats get returned %d\n", ret);
1067                 goto fail;
1068         }
1069         if ((unsigned int)ret > XSTATS_MAX)
1070                 printf("%s %d: more xstats available than space\n",
1071                                 __func__, __LINE__);
1072
1073         static const uint64_t queue_expected[] = {
1074                 3 /* rx */,
1075                 3 /* tx */,
1076                 0 /* drop */,
1077                 3 /* inflights */,
1078                 0, 0, 0, 0, /* iq 0, 1, 2, 3 used */
1079                 /* QID-to-Port: pinned_flows, packets */
1080                 0, 0,
1081                 0, 0,
1082                 1, 3,
1083                 0, 0,
1084         };
1085         for (i = 0; (signed int)i < ret; i++) {
1086                 if (queue_expected[i] != values[i]) {
1087                         printf(
1088                                 "%d, Error, xstat %d (id %d) %s : %"PRIu64
1089                                 ", expect %"PRIu64"\n",
1090                                 __LINE__, i, ids[i], xstats_names[i].name,
1091                                 values[i], queue_expected[i]);
1092                         goto fail;
1093                 }
1094         }
1095
1096         /* Reset the queue stats here */
1097         ret = rte_event_dev_xstats_reset(evdev,
1098                                         RTE_EVENT_DEV_XSTATS_QUEUE, 0,
1099                                         NULL,
1100                                         0);
1101
1102         /* Verify that the resetable stats are reset, and others are not */
1103         static const uint64_t queue_expected_zero[] = {
1104                 0 /* rx */,
1105                 0 /* tx */,
1106                 0 /* drop */,
1107                 3 /* inflight */,
1108                 0, 0, 0, 0, /* 4 iq used */
1109                 /* QID-to-Port: pinned_flows, packets */
1110                 0, 0,
1111                 0, 0,
1112                 1, 0,
1113                 0, 0,
1114         };
1115
1116         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE, 0,
1117                                         ids, values, num_stats);
1118         int fails = 0;
1119         for (i = 0; (signed int)i < ret; i++) {
1120                 if (queue_expected_zero[i] != values[i]) {
1121                         printf(
1122                                 "%d, Error, xstat %d (id %d) %s : %"PRIu64
1123                                 ", expect %"PRIu64"\n",
1124                                 __LINE__, i, ids[i], xstats_names[i].name,
1125                                 values[i], queue_expected_zero[i]);
1126                         fails++;
1127                 }
1128         }
1129         if (fails) {
1130                 printf("%d : %d of values were not as expected above\n",
1131                                 __LINE__, fails);
1132                 goto fail;
1133         }
1134
1135         cleanup(t);
1136         return 0;
1137
1138 fail:
1139         rte_event_dev_dump(0, stdout);
1140         cleanup(t);
1141         return -1;
1142 }
1143
1144
1145 static int
1146 xstats_id_abuse_tests(struct test *t)
1147 {
1148         int err;
1149         const uint32_t XSTATS_MAX = 1024;
1150         const uint32_t link_port = 2;
1151
1152         uint32_t ids[XSTATS_MAX];
1153         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
1154
1155         /* Create instance with 4 ports */
1156         if (init(t, 1, 4) < 0 ||
1157                         create_ports(t, 4) < 0 ||
1158                         create_atomic_qids(t, 1) < 0) {
1159                 printf("%d: Error initializing device\n", __LINE__);
1160                 goto fail;
1161         }
1162
1163         err = rte_event_port_link(evdev, t->port[link_port], NULL, NULL, 0);
1164         if (err != 1) {
1165                 printf("%d: error mapping lb qid\n", __LINE__);
1166                 goto fail;
1167         }
1168
1169         if (rte_event_dev_start(evdev) < 0) {
1170                 printf("%d: Error with start call\n", __LINE__);
1171                 goto fail;
1172         }
1173
1174         /* no test for device, as it ignores the port/q number */
1175         int num_stats = rte_event_dev_xstats_names_get(evdev,
1176                                         RTE_EVENT_DEV_XSTATS_PORT,
1177                                         UINT8_MAX-1, xstats_names, ids,
1178                                         XSTATS_MAX);
1179         if (num_stats != 0) {
1180                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1181                                 0, num_stats);
1182                 goto fail;
1183         }
1184
1185         num_stats = rte_event_dev_xstats_names_get(evdev,
1186                                         RTE_EVENT_DEV_XSTATS_QUEUE,
1187                                         UINT8_MAX-1, xstats_names, ids,
1188                                         XSTATS_MAX);
1189         if (num_stats != 0) {
1190                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1191                                 0, num_stats);
1192                 goto fail;
1193         }
1194
1195         cleanup(t);
1196         return 0;
1197 fail:
1198         cleanup(t);
1199         return -1;
1200 }
1201
1202 static int
1203 port_reconfig_credits(struct test *t)
1204 {
1205         if (init(t, 1, 1) < 0) {
1206                 printf("%d: Error initializing device\n", __LINE__);
1207                 return -1;
1208         }
1209
1210         uint32_t i;
1211         const uint32_t NUM_ITERS = 32;
1212         for (i = 0; i < NUM_ITERS; i++) {
1213                 const struct rte_event_queue_conf conf = {
1214                         .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1215                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1216                         .nb_atomic_flows = 1024,
1217                         .nb_atomic_order_sequences = 1024,
1218                 };
1219                 if (rte_event_queue_setup(evdev, 0, &conf) < 0) {
1220                         printf("%d: error creating qid\n", __LINE__);
1221                         return -1;
1222                 }
1223                 t->qid[0] = 0;
1224
1225                 static const struct rte_event_port_conf port_conf = {
1226                                 .new_event_threshold = 128,
1227                                 .dequeue_depth = 32,
1228                                 .enqueue_depth = 64,
1229                 };
1230                 if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
1231                         printf("%d Error setting up port\n", __LINE__);
1232                         return -1;
1233                 }
1234
1235                 int links = rte_event_port_link(evdev, 0, NULL, NULL, 0);
1236                 if (links != 1) {
1237                         printf("%d: error mapping lb qid\n", __LINE__);
1238                         goto fail;
1239                 }
1240
1241                 if (rte_event_dev_start(evdev) < 0) {
1242                         printf("%d: Error with start call\n", __LINE__);
1243                         goto fail;
1244                 }
1245
1246                 const uint32_t NPKTS = 1;
1247                 uint32_t j;
1248                 for (j = 0; j < NPKTS; j++) {
1249                         struct rte_event ev;
1250                         struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1251                         if (!arp) {
1252                                 printf("%d: gen of pkt failed\n", __LINE__);
1253                                 goto fail;
1254                         }
1255                         ev.queue_id = t->qid[0];
1256                         ev.op = RTE_EVENT_OP_NEW;
1257                         ev.mbuf = arp;
1258                         int err = rte_event_enqueue_burst(evdev, 0, &ev, 1);
1259                         if (err != 1) {
1260                                 printf("%d: Failed to enqueue\n", __LINE__);
1261                                 rte_event_dev_dump(0, stdout);
1262                                 goto fail;
1263                         }
1264                 }
1265
1266                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
1267
1268                 struct rte_event ev[NPKTS];
1269                 int deq = rte_event_dequeue_burst(evdev, t->port[0], ev,
1270                                                         NPKTS, 0);
1271                 if (deq != 1)
1272                         printf("%d error; no packet dequeued\n", __LINE__);
1273
1274                 /* let cleanup below stop the device on last iter */
1275                 if (i != NUM_ITERS-1)
1276                         rte_event_dev_stop(evdev);
1277         }
1278
1279         cleanup(t);
1280         return 0;
1281 fail:
1282         cleanup(t);
1283         return -1;
1284 }
1285
1286 static int
1287 port_single_lb_reconfig(struct test *t)
1288 {
1289         if (init(t, 2, 2) < 0) {
1290                 printf("%d: Error initializing device\n", __LINE__);
1291                 goto fail;
1292         }
1293
1294         static const struct rte_event_queue_conf conf_lb_atomic = {
1295                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1296                 .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1297                 .nb_atomic_flows = 1024,
1298                 .nb_atomic_order_sequences = 1024,
1299         };
1300         if (rte_event_queue_setup(evdev, 0, &conf_lb_atomic) < 0) {
1301                 printf("%d: error creating qid\n", __LINE__);
1302                 goto fail;
1303         }
1304
1305         static const struct rte_event_queue_conf conf_single_link = {
1306                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1307                 .event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK,
1308         };
1309         if (rte_event_queue_setup(evdev, 1, &conf_single_link) < 0) {
1310                 printf("%d: error creating qid\n", __LINE__);
1311                 goto fail;
1312         }
1313
1314         struct rte_event_port_conf port_conf = {
1315                 .new_event_threshold = 128,
1316                 .dequeue_depth = 32,
1317                 .enqueue_depth = 64,
1318         };
1319         if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
1320                 printf("%d Error setting up port\n", __LINE__);
1321                 goto fail;
1322         }
1323         if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
1324                 printf("%d Error setting up port\n", __LINE__);
1325                 goto fail;
1326         }
1327
1328         /* link port to lb queue */
1329         uint8_t queue_id = 0;
1330         if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
1331                 printf("%d: error creating link for qid\n", __LINE__);
1332                 goto fail;
1333         }
1334
1335         int ret = rte_event_port_unlink(evdev, 0, &queue_id, 1);
1336         if (ret != 1) {
1337                 printf("%d: Error unlinking lb port\n", __LINE__);
1338                 goto fail;
1339         }
1340
1341         queue_id = 1;
1342         if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
1343                 printf("%d: error creating link for qid\n", __LINE__);
1344                 goto fail;
1345         }
1346
1347         queue_id = 0;
1348         int err = rte_event_port_link(evdev, 1, &queue_id, NULL, 1);
1349         if (err != 1) {
1350                 printf("%d: error mapping lb qid\n", __LINE__);
1351                 goto fail;
1352         }
1353
1354         if (rte_event_dev_start(evdev) < 0) {
1355                 printf("%d: Error with start call\n", __LINE__);
1356                 goto fail;
1357         }
1358
1359         cleanup(t);
1360         return 0;
1361 fail:
1362         cleanup(t);
1363         return -1;
1364 }
1365
1366 static int
1367 xstats_brute_force(struct test *t)
1368 {
1369         uint32_t i;
1370         const uint32_t XSTATS_MAX = 1024;
1371         uint32_t ids[XSTATS_MAX];
1372         uint64_t values[XSTATS_MAX];
1373         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
1374
1375
1376         /* Create instance with 4 ports */
1377         if (init(t, 1, 4) < 0 ||
1378                         create_ports(t, 4) < 0 ||
1379                         create_atomic_qids(t, 1) < 0) {
1380                 printf("%d: Error initializing device\n", __LINE__);
1381                 return -1;
1382         }
1383
1384         int err = rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1385         if (err != 1) {
1386                 printf("%d: error mapping lb qid\n", __LINE__);
1387                 goto fail;
1388         }
1389
1390         if (rte_event_dev_start(evdev) < 0) {
1391                 printf("%d: Error with start call\n", __LINE__);
1392                 goto fail;
1393         }
1394
1395         for (i = 0; i < XSTATS_MAX; i++)
1396                 ids[i] = i;
1397
1398         for (i = 0; i < 3; i++) {
1399                 uint32_t mode = RTE_EVENT_DEV_XSTATS_DEVICE + i;
1400                 uint32_t j;
1401                 for (j = 0; j < UINT8_MAX; j++) {
1402                         rte_event_dev_xstats_names_get(evdev, mode,
1403                                 j, xstats_names, ids, XSTATS_MAX);
1404
1405                         rte_event_dev_xstats_get(evdev, mode, j, ids,
1406                                                  values, XSTATS_MAX);
1407                 }
1408         }
1409
1410         cleanup(t);
1411         return 0;
1412 fail:
1413         cleanup(t);
1414         return -1;
1415 }
1416
1417 static int
1418 xstats_id_reset_tests(struct test *t)
1419 {
1420         const int wrk_enq = 2;
1421         int err;
1422
1423         /* Create instance with 4 ports */
1424         if (init(t, 1, 4) < 0 ||
1425                         create_ports(t, 4) < 0 ||
1426                         create_atomic_qids(t, 1) < 0) {
1427                 printf("%d: Error initializing device\n", __LINE__);
1428                 return -1;
1429         }
1430
1431         /* CQ mapping to QID */
1432         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
1433         if (err != 1) {
1434                 printf("%d: error mapping lb qid\n", __LINE__);
1435                 goto fail;
1436         }
1437
1438         if (rte_event_dev_start(evdev) < 0) {
1439                 printf("%d: Error with start call\n", __LINE__);
1440                 goto fail;
1441         }
1442
1443 #define XSTATS_MAX 1024
1444         int ret;
1445         uint32_t i;
1446         uint32_t ids[XSTATS_MAX];
1447         uint64_t values[XSTATS_MAX];
1448         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
1449
1450         for (i = 0; i < XSTATS_MAX; i++)
1451                 ids[i] = i;
1452
1453 #define NUM_DEV_STATS 6
1454         /* Device names / values */
1455         int num_stats = rte_event_dev_xstats_names_get(evdev,
1456                                         RTE_EVENT_DEV_XSTATS_DEVICE,
1457                                         0, xstats_names, ids, XSTATS_MAX);
1458         if (num_stats != NUM_DEV_STATS) {
1459                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1460                                 NUM_DEV_STATS, num_stats);
1461                 goto fail;
1462         }
1463         ret = rte_event_dev_xstats_get(evdev,
1464                                         RTE_EVENT_DEV_XSTATS_DEVICE,
1465                                         0, ids, values, num_stats);
1466         if (ret != NUM_DEV_STATS) {
1467                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1468                                 NUM_DEV_STATS, ret);
1469                 goto fail;
1470         }
1471
1472 #define NPKTS 7
1473         for (i = 0; i < NPKTS; i++) {
1474                 struct rte_event ev;
1475                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1476                 if (!arp) {
1477                         printf("%d: gen of pkt failed\n", __LINE__);
1478                         goto fail;
1479                 }
1480                 ev.queue_id = t->qid[i];
1481                 ev.op = RTE_EVENT_OP_NEW;
1482                 ev.mbuf = arp;
1483                 arp->seqn = i;
1484
1485                 int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
1486                 if (err != 1) {
1487                         printf("%d: Failed to enqueue\n", __LINE__);
1488                         goto fail;
1489                 }
1490         }
1491
1492         rte_service_run_iter_on_app_lcore(t->service_id, 1);
1493
1494         static const char * const dev_names[] = {
1495                 "dev_rx", "dev_tx", "dev_drop", "dev_sched_calls",
1496                 "dev_sched_no_iq_enq", "dev_sched_no_cq_enq",
1497         };
1498         uint64_t dev_expected[] = {NPKTS, NPKTS, 0, 1, 0, 0};
1499         for (i = 0; (int)i < ret; i++) {
1500                 unsigned int id;
1501                 uint64_t val = rte_event_dev_xstats_by_name_get(evdev,
1502                                                                 dev_names[i],
1503                                                                 &id);
1504                 if (id != i) {
1505                         printf("%d: %s id incorrect, expected %d got %d\n",
1506                                         __LINE__, dev_names[i], i, id);
1507                         goto fail;
1508                 }
1509                 if (val != dev_expected[i]) {
1510                         printf("%d: %s value incorrect, expected %"
1511                                 PRIu64" got %d\n", __LINE__, dev_names[i],
1512                                 dev_expected[i], id);
1513                         goto fail;
1514                 }
1515                 /* reset to zero */
1516                 int reset_ret = rte_event_dev_xstats_reset(evdev,
1517                                                 RTE_EVENT_DEV_XSTATS_DEVICE, 0,
1518                                                 &id,
1519                                                 1);
1520                 if (reset_ret) {
1521                         printf("%d: failed to reset successfully\n", __LINE__);
1522                         goto fail;
1523                 }
1524                 dev_expected[i] = 0;
1525                 /* check value again */
1526                 val = rte_event_dev_xstats_by_name_get(evdev, dev_names[i], 0);
1527                 if (val != dev_expected[i]) {
1528                         printf("%d: %s value incorrect, expected %"PRIu64
1529                                 " got %"PRIu64"\n", __LINE__, dev_names[i],
1530                                 dev_expected[i], val);
1531                         goto fail;
1532                 }
1533         };
1534
1535 /* 48 is stat offset from start of the devices whole xstats.
1536  * This WILL break every time we add a statistic to a port
1537  * or the device, but there is no other way to test
1538  */
1539 #define PORT_OFF 48
1540 /* num stats for the tested port. CQ size adds more stats to a port */
1541 #define NUM_PORT_STATS 21
1542 /* the port to test. */
1543 #define PORT 2
1544         num_stats = rte_event_dev_xstats_names_get(evdev,
1545                                         RTE_EVENT_DEV_XSTATS_PORT, PORT,
1546                                         xstats_names, ids, XSTATS_MAX);
1547         if (num_stats != NUM_PORT_STATS) {
1548                 printf("%d: expected %d stats, got return %d\n",
1549                         __LINE__, NUM_PORT_STATS, num_stats);
1550                 goto fail;
1551         }
1552         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_PORT, PORT,
1553                                         ids, values, num_stats);
1554
1555         if (ret != NUM_PORT_STATS) {
1556                 printf("%d: expected %d stats, got return %d\n",
1557                                 __LINE__, NUM_PORT_STATS, ret);
1558                 goto fail;
1559         }
1560         static const char * const port_names[] = {
1561                 "port_2_rx",
1562                 "port_2_tx",
1563                 "port_2_drop",
1564                 "port_2_inflight",
1565                 "port_2_avg_pkt_cycles",
1566                 "port_2_credits",
1567                 "port_2_rx_ring_used",
1568                 "port_2_rx_ring_free",
1569                 "port_2_cq_ring_used",
1570                 "port_2_cq_ring_free",
1571                 "port_2_dequeue_calls",
1572                 "port_2_dequeues_returning_0",
1573                 "port_2_dequeues_returning_1-4",
1574                 "port_2_dequeues_returning_5-8",
1575                 "port_2_dequeues_returning_9-12",
1576                 "port_2_dequeues_returning_13-16",
1577                 "port_2_dequeues_returning_17-20",
1578                 "port_2_dequeues_returning_21-24",
1579                 "port_2_dequeues_returning_25-28",
1580                 "port_2_dequeues_returning_29-32",
1581                 "port_2_dequeues_returning_33-36",
1582         };
1583         uint64_t port_expected[] = {
1584                 0, /* rx */
1585                 NPKTS, /* tx */
1586                 0, /* drop */
1587                 NPKTS, /* inflight */
1588                 0, /* avg pkt cycles */
1589                 0, /* credits */
1590                 0, /* rx ring used */
1591                 4096, /* rx ring free */
1592                 NPKTS,  /* cq ring used */
1593                 25, /* cq ring free */
1594                 0, /* dequeue zero calls */
1595                 0, 0, 0, 0, 0, /* 10 dequeue buckets */
1596                 0, 0, 0, 0, 0,
1597         };
1598         uint64_t port_expected_zero[] = {
1599                 0, /* rx */
1600                 0, /* tx */
1601                 0, /* drop */
1602                 NPKTS, /* inflight */
1603                 0, /* avg pkt cycles */
1604                 0, /* credits */
1605                 0, /* rx ring used */
1606                 4096, /* rx ring free */
1607                 NPKTS,  /* cq ring used */
1608                 25, /* cq ring free */
1609                 0, /* dequeue zero calls */
1610                 0, 0, 0, 0, 0, /* 10 dequeue buckets */
1611                 0, 0, 0, 0, 0,
1612         };
1613         if (RTE_DIM(port_expected) != NUM_PORT_STATS ||
1614                         RTE_DIM(port_names) != NUM_PORT_STATS) {
1615                 printf("%d: port array of wrong size\n", __LINE__);
1616                 goto fail;
1617         }
1618
1619         int failed = 0;
1620         for (i = 0; (int)i < ret; i++) {
1621                 unsigned int id;
1622                 uint64_t val = rte_event_dev_xstats_by_name_get(evdev,
1623                                                                 port_names[i],
1624                                                                 &id);
1625                 if (id != i + PORT_OFF) {
1626                         printf("%d: %s id incorrect, expected %d got %d\n",
1627                                         __LINE__, port_names[i], i+PORT_OFF,
1628                                         id);
1629                         failed = 1;
1630                 }
1631                 if (val != port_expected[i]) {
1632                         printf("%d: %s value incorrect, expected %"PRIu64
1633                                 " got %d\n", __LINE__, port_names[i],
1634                                 port_expected[i], id);
1635                         failed = 1;
1636                 }
1637                 /* reset to zero */
1638                 int reset_ret = rte_event_dev_xstats_reset(evdev,
1639                                                 RTE_EVENT_DEV_XSTATS_PORT, PORT,
1640                                                 &id,
1641                                                 1);
1642                 if (reset_ret) {
1643                         printf("%d: failed to reset successfully\n", __LINE__);
1644                         failed = 1;
1645                 }
1646                 /* check value again */
1647                 val = rte_event_dev_xstats_by_name_get(evdev, port_names[i], 0);
1648                 if (val != port_expected_zero[i]) {
1649                         printf("%d: %s value incorrect, expected %"PRIu64
1650                                 " got %"PRIu64"\n", __LINE__, port_names[i],
1651                                 port_expected_zero[i], val);
1652                         failed = 1;
1653                 }
1654         };
1655         if (failed)
1656                 goto fail;
1657
1658 /* num queue stats */
1659 #define NUM_Q_STATS 16
1660 /* queue offset from start of the devices whole xstats.
1661  * This will break every time we add a statistic to a device/port/queue
1662  */
1663 #define QUEUE_OFF 90
1664         const uint32_t queue = 0;
1665         num_stats = rte_event_dev_xstats_names_get(evdev,
1666                                         RTE_EVENT_DEV_XSTATS_QUEUE, queue,
1667                                         xstats_names, ids, XSTATS_MAX);
1668         if (num_stats != NUM_Q_STATS) {
1669                 printf("%d: expected %d stats, got return %d\n",
1670                         __LINE__, NUM_Q_STATS, num_stats);
1671                 goto fail;
1672         }
1673         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE,
1674                                         queue, ids, values, num_stats);
1675         if (ret != NUM_Q_STATS) {
1676                 printf("%d: expected 21 stats, got return %d\n", __LINE__, ret);
1677                 goto fail;
1678         }
1679         static const char * const queue_names[] = {
1680                 "qid_0_rx",
1681                 "qid_0_tx",
1682                 "qid_0_drop",
1683                 "qid_0_inflight",
1684                 "qid_0_iq_0_used",
1685                 "qid_0_iq_1_used",
1686                 "qid_0_iq_2_used",
1687                 "qid_0_iq_3_used",
1688                 "qid_0_port_0_pinned_flows",
1689                 "qid_0_port_0_packets",
1690                 "qid_0_port_1_pinned_flows",
1691                 "qid_0_port_1_packets",
1692                 "qid_0_port_2_pinned_flows",
1693                 "qid_0_port_2_packets",
1694                 "qid_0_port_3_pinned_flows",
1695                 "qid_0_port_3_packets",
1696         };
1697         uint64_t queue_expected[] = {
1698                 7, /* rx */
1699                 7, /* tx */
1700                 0, /* drop */
1701                 7, /* inflight */
1702                 0, /* iq 0 used */
1703                 0, /* iq 1 used */
1704                 0, /* iq 2 used */
1705                 0, /* iq 3 used */
1706                 /* QID-to-Port: pinned_flows, packets */
1707                 0, 0,
1708                 0, 0,
1709                 1, 7,
1710                 0, 0,
1711         };
1712         uint64_t queue_expected_zero[] = {
1713                 0, /* rx */
1714                 0, /* tx */
1715                 0, /* drop */
1716                 7, /* inflight */
1717                 0, /* iq 0 used */
1718                 0, /* iq 1 used */
1719                 0, /* iq 2 used */
1720                 0, /* iq 3 used */
1721                 /* QID-to-Port: pinned_flows, packets */
1722                 0, 0,
1723                 0, 0,
1724                 1, 0,
1725                 0, 0,
1726         };
1727         if (RTE_DIM(queue_expected) != NUM_Q_STATS ||
1728                         RTE_DIM(queue_expected_zero) != NUM_Q_STATS ||
1729                         RTE_DIM(queue_names) != NUM_Q_STATS) {
1730                 printf("%d : queue array of wrong size\n", __LINE__);
1731                 goto fail;
1732         }
1733
1734         failed = 0;
1735         for (i = 0; (int)i < ret; i++) {
1736                 unsigned int id;
1737                 uint64_t val = rte_event_dev_xstats_by_name_get(evdev,
1738                                                                 queue_names[i],
1739                                                                 &id);
1740                 if (id != i + QUEUE_OFF) {
1741                         printf("%d: %s id incorrect, expected %d got %d\n",
1742                                         __LINE__, queue_names[i], i+QUEUE_OFF,
1743                                         id);
1744                         failed = 1;
1745                 }
1746                 if (val != queue_expected[i]) {
1747                         printf("%d: %d: %s value , expected %"PRIu64
1748                                 " got %"PRIu64"\n", i, __LINE__,
1749                                 queue_names[i], queue_expected[i], val);
1750                         failed = 1;
1751                 }
1752                 /* reset to zero */
1753                 int reset_ret = rte_event_dev_xstats_reset(evdev,
1754                                                 RTE_EVENT_DEV_XSTATS_QUEUE,
1755                                                 queue, &id, 1);
1756                 if (reset_ret) {
1757                         printf("%d: failed to reset successfully\n", __LINE__);
1758                         failed = 1;
1759                 }
1760                 /* check value again */
1761                 val = rte_event_dev_xstats_by_name_get(evdev, queue_names[i],
1762                                                         0);
1763                 if (val != queue_expected_zero[i]) {
1764                         printf("%d: %s value incorrect, expected %"PRIu64
1765                                 " got %"PRIu64"\n", __LINE__, queue_names[i],
1766                                 queue_expected_zero[i], val);
1767                         failed = 1;
1768                 }
1769         };
1770
1771         if (failed)
1772                 goto fail;
1773
1774         cleanup(t);
1775         return 0;
1776 fail:
1777         cleanup(t);
1778         return -1;
1779 }
1780
1781 static int
1782 ordered_reconfigure(struct test *t)
1783 {
1784         if (init(t, 1, 1) < 0 ||
1785                         create_ports(t, 1) < 0) {
1786                 printf("%d: Error initializing device\n", __LINE__);
1787                 return -1;
1788         }
1789
1790         const struct rte_event_queue_conf conf = {
1791                         .schedule_type = RTE_SCHED_TYPE_ORDERED,
1792                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1793                         .nb_atomic_flows = 1024,
1794                         .nb_atomic_order_sequences = 1024,
1795         };
1796
1797         if (rte_event_queue_setup(evdev, 0, &conf) < 0) {
1798                 printf("%d: error creating qid\n", __LINE__);
1799                 goto failed;
1800         }
1801
1802         if (rte_event_queue_setup(evdev, 0, &conf) < 0) {
1803                 printf("%d: error creating qid, for 2nd time\n", __LINE__);
1804                 goto failed;
1805         }
1806
1807         rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1808         if (rte_event_dev_start(evdev) < 0) {
1809                 printf("%d: Error with start call\n", __LINE__);
1810                 return -1;
1811         }
1812
1813         cleanup(t);
1814         return 0;
1815 failed:
1816         cleanup(t);
1817         return -1;
1818 }
1819
1820 static int
1821 qid_priorities(struct test *t)
1822 {
1823         /* Test works by having a CQ with enough empty space for all packets,
1824          * and enqueueing 3 packets to 3 QIDs. They must return based on the
1825          * priority of the QID, not the ingress order, to pass the test
1826          */
1827         unsigned int i;
1828         /* Create instance with 1 ports, and 3 qids */
1829         if (init(t, 3, 1) < 0 ||
1830                         create_ports(t, 1) < 0) {
1831                 printf("%d: Error initializing device\n", __LINE__);
1832                 return -1;
1833         }
1834
1835         for (i = 0; i < 3; i++) {
1836                 /* Create QID */
1837                 const struct rte_event_queue_conf conf = {
1838                         .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1839                         /* increase priority (0 == highest), as we go */
1840                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL - i,
1841                         .nb_atomic_flows = 1024,
1842                         .nb_atomic_order_sequences = 1024,
1843                 };
1844
1845                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
1846                         printf("%d: error creating qid %d\n", __LINE__, i);
1847                         return -1;
1848                 }
1849                 t->qid[i] = i;
1850         }
1851         t->nb_qids = i;
1852         /* map all QIDs to port */
1853         rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1854
1855         if (rte_event_dev_start(evdev) < 0) {
1856                 printf("%d: Error with start call\n", __LINE__);
1857                 return -1;
1858         }
1859
1860         /* enqueue 3 packets, setting seqn and QID to check priority */
1861         for (i = 0; i < 3; i++) {
1862                 struct rte_event ev;
1863                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1864                 if (!arp) {
1865                         printf("%d: gen of pkt failed\n", __LINE__);
1866                         return -1;
1867                 }
1868                 ev.queue_id = t->qid[i];
1869                 ev.op = RTE_EVENT_OP_NEW;
1870                 ev.mbuf = arp;
1871                 arp->seqn = i;
1872
1873                 int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
1874                 if (err != 1) {
1875                         printf("%d: Failed to enqueue\n", __LINE__);
1876                         return -1;
1877                 }
1878         }
1879
1880         rte_service_run_iter_on_app_lcore(t->service_id, 1);
1881
1882         /* dequeue packets, verify priority was upheld */
1883         struct rte_event ev[32];
1884         uint32_t deq_pkts =
1885                 rte_event_dequeue_burst(evdev, t->port[0], ev, 32, 0);
1886         if (deq_pkts != 3) {
1887                 printf("%d: failed to deq packets\n", __LINE__);
1888                 rte_event_dev_dump(evdev, stdout);
1889                 return -1;
1890         }
1891         for (i = 0; i < 3; i++) {
1892                 if (ev[i].mbuf->seqn != 2-i) {
1893                         printf(
1894                                 "%d: qid priority test: seqn %d incorrectly prioritized\n",
1895                                         __LINE__, i);
1896                 }
1897         }
1898
1899         cleanup(t);
1900         return 0;
1901 }
1902
1903 static int
1904 unlink_in_progress(struct test *t)
1905 {
1906         /* Test unlinking API, in particular that when an unlink request has
1907          * not yet been seen by the scheduler thread, that the
1908          * unlink_in_progress() function returns the number of unlinks.
1909          */
1910         unsigned int i;
1911         /* Create instance with 1 ports, and 3 qids */
1912         if (init(t, 3, 1) < 0 ||
1913                         create_ports(t, 1) < 0) {
1914                 printf("%d: Error initializing device\n", __LINE__);
1915                 return -1;
1916         }
1917
1918         for (i = 0; i < 3; i++) {
1919                 /* Create QID */
1920                 const struct rte_event_queue_conf conf = {
1921                         .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1922                         /* increase priority (0 == highest), as we go */
1923                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL - i,
1924                         .nb_atomic_flows = 1024,
1925                         .nb_atomic_order_sequences = 1024,
1926                 };
1927
1928                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
1929                         printf("%d: error creating qid %d\n", __LINE__, i);
1930                         return -1;
1931                 }
1932                 t->qid[i] = i;
1933         }
1934         t->nb_qids = i;
1935         /* map all QIDs to port */
1936         rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1937
1938         if (rte_event_dev_start(evdev) < 0) {
1939                 printf("%d: Error with start call\n", __LINE__);
1940                 return -1;
1941         }
1942
1943         /* unlink all ports to have outstanding unlink requests */
1944         int ret = rte_event_port_unlink(evdev, t->port[0], NULL, 0);
1945         if (ret < 0) {
1946                 printf("%d: Failed to unlink queues\n", __LINE__);
1947                 return -1;
1948         }
1949
1950         /* get active unlinks here, expect 3 */
1951         int unlinks_in_progress =
1952                 rte_event_port_unlinks_in_progress(evdev, t->port[0]);
1953         if (unlinks_in_progress != 3) {
1954                 printf("%d: Expected num unlinks in progress == 3, got %d\n",
1955                                 __LINE__, unlinks_in_progress);
1956                 return -1;
1957         }
1958
1959         /* run scheduler service on this thread to ack the unlinks */
1960         rte_service_run_iter_on_app_lcore(t->service_id, 1);
1961
1962         /* active unlinks expected as 0 as scheduler thread has acked */
1963         unlinks_in_progress =
1964                 rte_event_port_unlinks_in_progress(evdev, t->port[0]);
1965         if (unlinks_in_progress != 0) {
1966                 printf("%d: Expected num unlinks in progress == 0, got %d\n",
1967                                 __LINE__, unlinks_in_progress);
1968         }
1969
1970         cleanup(t);
1971         return 0;
1972 }
1973
1974 static int
1975 load_balancing(struct test *t)
1976 {
1977         const int rx_enq = 0;
1978         int err;
1979         uint32_t i;
1980
1981         if (init(t, 1, 4) < 0 ||
1982                         create_ports(t, 4) < 0 ||
1983                         create_atomic_qids(t, 1) < 0) {
1984                 printf("%d: Error initializing device\n", __LINE__);
1985                 return -1;
1986         }
1987
1988         for (i = 0; i < 3; i++) {
1989                 /* map port 1 - 3 inclusive */
1990                 if (rte_event_port_link(evdev, t->port[i+1], &t->qid[0],
1991                                 NULL, 1) != 1) {
1992                         printf("%d: error mapping qid to port %d\n",
1993                                         __LINE__, i);
1994                         return -1;
1995                 }
1996         }
1997
1998         if (rte_event_dev_start(evdev) < 0) {
1999                 printf("%d: Error with start call\n", __LINE__);
2000                 return -1;
2001         }
2002
2003         /************** FORWARD ****************/
2004         /*
2005          * Create a set of flows that test the load-balancing operation of the
2006          * implementation. Fill CQ 0 and 1 with flows 0 and 1, and test
2007          * with a new flow, which should be sent to the 3rd mapped CQ
2008          */
2009         static uint32_t flows[] = {0, 1, 1, 0, 0, 2, 2, 0, 2};
2010
2011         for (i = 0; i < RTE_DIM(flows); i++) {
2012                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2013                 if (!arp) {
2014                         printf("%d: gen of pkt failed\n", __LINE__);
2015                         return -1;
2016                 }
2017
2018                 struct rte_event ev = {
2019                                 .op = RTE_EVENT_OP_NEW,
2020                                 .queue_id = t->qid[0],
2021                                 .flow_id = flows[i],
2022                                 .mbuf = arp,
2023                 };
2024                 /* generate pkt and enqueue */
2025                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2026                 if (err != 1) {
2027                         printf("%d: Failed to enqueue\n", __LINE__);
2028                         return -1;
2029                 }
2030         }
2031
2032         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2033
2034         struct test_event_dev_stats stats;
2035         err = test_event_dev_stats_get(evdev, &stats);
2036         if (err) {
2037                 printf("%d: failed to get stats\n", __LINE__);
2038                 return -1;
2039         }
2040
2041         if (stats.port_inflight[1] != 4) {
2042                 printf("%d:%s: port 1 inflight not correct\n", __LINE__,
2043                                 __func__);
2044                 return -1;
2045         }
2046         if (stats.port_inflight[2] != 2) {
2047                 printf("%d:%s: port 2 inflight not correct\n", __LINE__,
2048                                 __func__);
2049                 return -1;
2050         }
2051         if (stats.port_inflight[3] != 3) {
2052                 printf("%d:%s: port 3 inflight not correct\n", __LINE__,
2053                                 __func__);
2054                 return -1;
2055         }
2056
2057         cleanup(t);
2058         return 0;
2059 }
2060
2061 static int
2062 load_balancing_history(struct test *t)
2063 {
2064         struct test_event_dev_stats stats = {0};
2065         const int rx_enq = 0;
2066         int err;
2067         uint32_t i;
2068
2069         /* Create instance with 1 atomic QID going to 3 ports + 1 prod port */
2070         if (init(t, 1, 4) < 0 ||
2071                         create_ports(t, 4) < 0 ||
2072                         create_atomic_qids(t, 1) < 0)
2073                 return -1;
2074
2075         /* CQ mapping to QID */
2076         if (rte_event_port_link(evdev, t->port[1], &t->qid[0], NULL, 1) != 1) {
2077                 printf("%d: error mapping port 1 qid\n", __LINE__);
2078                 return -1;
2079         }
2080         if (rte_event_port_link(evdev, t->port[2], &t->qid[0], NULL, 1) != 1) {
2081                 printf("%d: error mapping port 2 qid\n", __LINE__);
2082                 return -1;
2083         }
2084         if (rte_event_port_link(evdev, t->port[3], &t->qid[0], NULL, 1) != 1) {
2085                 printf("%d: error mapping port 3 qid\n", __LINE__);
2086                 return -1;
2087         }
2088         if (rte_event_dev_start(evdev) < 0) {
2089                 printf("%d: Error with start call\n", __LINE__);
2090                 return -1;
2091         }
2092
2093         /*
2094          * Create a set of flows that test the load-balancing operation of the
2095          * implementation. Fill CQ 0, 1 and 2 with flows 0, 1 and 2, drop
2096          * the packet from CQ 0, send in a new set of flows. Ensure that:
2097          *  1. The new flow 3 gets into the empty CQ0
2098          *  2. packets for existing flow gets added into CQ1
2099          *  3. Next flow 0 pkt is now onto CQ2, since CQ0 and CQ1 now contain
2100          *     more outstanding pkts
2101          *
2102          *  This test makes sure that when a flow ends (i.e. all packets
2103          *  have been completed for that flow), that the flow can be moved
2104          *  to a different CQ when new packets come in for that flow.
2105          */
2106         static uint32_t flows1[] = {0, 1, 1, 2};
2107
2108         for (i = 0; i < RTE_DIM(flows1); i++) {
2109                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2110                 struct rte_event ev = {
2111                                 .flow_id = flows1[i],
2112                                 .op = RTE_EVENT_OP_NEW,
2113                                 .queue_id = t->qid[0],
2114                                 .event_type = RTE_EVENT_TYPE_CPU,
2115                                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
2116                                 .mbuf = arp
2117                 };
2118
2119                 if (!arp) {
2120                         printf("%d: gen of pkt failed\n", __LINE__);
2121                         return -1;
2122                 }
2123                 arp->hash.rss = flows1[i];
2124                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2125                 if (err != 1) {
2126                         printf("%d: Failed to enqueue\n", __LINE__);
2127                         return -1;
2128                 }
2129         }
2130
2131         /* call the scheduler */
2132         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2133
2134         /* Dequeue the flow 0 packet from port 1, so that we can then drop */
2135         struct rte_event ev;
2136         if (!rte_event_dequeue_burst(evdev, t->port[1], &ev, 1, 0)) {
2137                 printf("%d: failed to dequeue\n", __LINE__);
2138                 return -1;
2139         }
2140         if (ev.mbuf->hash.rss != flows1[0]) {
2141                 printf("%d: unexpected flow received\n", __LINE__);
2142                 return -1;
2143         }
2144
2145         /* drop the flow 0 packet from port 1 */
2146         rte_event_enqueue_burst(evdev, t->port[1], &release_ev, 1);
2147
2148         /* call the scheduler */
2149         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2150
2151         /*
2152          * Set up the next set of flows, first a new flow to fill up
2153          * CQ 0, so that the next flow 0 packet should go to CQ2
2154          */
2155         static uint32_t flows2[] = { 3, 3, 3, 1, 1, 0 };
2156
2157         for (i = 0; i < RTE_DIM(flows2); i++) {
2158                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2159                 struct rte_event ev = {
2160                                 .flow_id = flows2[i],
2161                                 .op = RTE_EVENT_OP_NEW,
2162                                 .queue_id = t->qid[0],
2163                                 .event_type = RTE_EVENT_TYPE_CPU,
2164                                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
2165                                 .mbuf = arp
2166                 };
2167
2168                 if (!arp) {
2169                         printf("%d: gen of pkt failed\n", __LINE__);
2170                         return -1;
2171                 }
2172                 arp->hash.rss = flows2[i];
2173
2174                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2175                 if (err != 1) {
2176                         printf("%d: Failed to enqueue\n", __LINE__);
2177                         return -1;
2178                 }
2179         }
2180
2181         /* schedule */
2182         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2183
2184         err = test_event_dev_stats_get(evdev, &stats);
2185         if (err) {
2186                 printf("%d:failed to get stats\n", __LINE__);
2187                 return -1;
2188         }
2189
2190         /*
2191          * Now check the resulting inflights on each port.
2192          */
2193         if (stats.port_inflight[1] != 3) {
2194                 printf("%d:%s: port 1 inflight not correct\n", __LINE__,
2195                                 __func__);
2196                 printf("Inflights, ports 1, 2, 3: %u, %u, %u\n",
2197                                 (unsigned int)stats.port_inflight[1],
2198                                 (unsigned int)stats.port_inflight[2],
2199                                 (unsigned int)stats.port_inflight[3]);
2200                 return -1;
2201         }
2202         if (stats.port_inflight[2] != 4) {
2203                 printf("%d:%s: port 2 inflight not correct\n", __LINE__,
2204                                 __func__);
2205                 printf("Inflights, ports 1, 2, 3: %u, %u, %u\n",
2206                                 (unsigned int)stats.port_inflight[1],
2207                                 (unsigned int)stats.port_inflight[2],
2208                                 (unsigned int)stats.port_inflight[3]);
2209                 return -1;
2210         }
2211         if (stats.port_inflight[3] != 2) {
2212                 printf("%d:%s: port 3 inflight not correct\n", __LINE__,
2213                                 __func__);
2214                 printf("Inflights, ports 1, 2, 3: %u, %u, %u\n",
2215                                 (unsigned int)stats.port_inflight[1],
2216                                 (unsigned int)stats.port_inflight[2],
2217                                 (unsigned int)stats.port_inflight[3]);
2218                 return -1;
2219         }
2220
2221         for (i = 1; i <= 3; i++) {
2222                 struct rte_event ev;
2223                 while (rte_event_dequeue_burst(evdev, i, &ev, 1, 0))
2224                         rte_event_enqueue_burst(evdev, i, &release_ev, 1);
2225         }
2226         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2227
2228         cleanup(t);
2229         return 0;
2230 }
2231
2232 static int
2233 invalid_qid(struct test *t)
2234 {
2235         struct test_event_dev_stats stats;
2236         const int rx_enq = 0;
2237         int err;
2238         uint32_t i;
2239
2240         if (init(t, 1, 4) < 0 ||
2241                         create_ports(t, 4) < 0 ||
2242                         create_atomic_qids(t, 1) < 0) {
2243                 printf("%d: Error initializing device\n", __LINE__);
2244                 return -1;
2245         }
2246
2247         /* CQ mapping to QID */
2248         for (i = 0; i < 4; i++) {
2249                 err = rte_event_port_link(evdev, t->port[i], &t->qid[0],
2250                                 NULL, 1);
2251                 if (err != 1) {
2252                         printf("%d: error mapping port 1 qid\n", __LINE__);
2253                         return -1;
2254                 }
2255         }
2256
2257         if (rte_event_dev_start(evdev) < 0) {
2258                 printf("%d: Error with start call\n", __LINE__);
2259                 return -1;
2260         }
2261
2262         /*
2263          * Send in a packet with an invalid qid to the scheduler.
2264          * We should see the packed enqueued OK, but the inflights for
2265          * that packet should not be incremented, and the rx_dropped
2266          * should be incremented.
2267          */
2268         static uint32_t flows1[] = {20};
2269
2270         for (i = 0; i < RTE_DIM(flows1); i++) {
2271                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2272                 if (!arp) {
2273                         printf("%d: gen of pkt failed\n", __LINE__);
2274                         return -1;
2275                 }
2276
2277                 struct rte_event ev = {
2278                                 .op = RTE_EVENT_OP_NEW,
2279                                 .queue_id = t->qid[0] + flows1[i],
2280                                 .flow_id = i,
2281                                 .mbuf = arp,
2282                 };
2283                 /* generate pkt and enqueue */
2284                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2285                 if (err != 1) {
2286                         printf("%d: Failed to enqueue\n", __LINE__);
2287                         return -1;
2288                 }
2289         }
2290
2291         /* call the scheduler */
2292         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2293
2294         err = test_event_dev_stats_get(evdev, &stats);
2295         if (err) {
2296                 printf("%d: failed to get stats\n", __LINE__);
2297                 return -1;
2298         }
2299
2300         /*
2301          * Now check the resulting inflights on the port, and the rx_dropped.
2302          */
2303         if (stats.port_inflight[0] != 0) {
2304                 printf("%d:%s: port 1 inflight count not correct\n", __LINE__,
2305                                 __func__);
2306                 rte_event_dev_dump(evdev, stdout);
2307                 return -1;
2308         }
2309         if (stats.port_rx_dropped[0] != 1) {
2310                 printf("%d:%s: port 1 drops\n", __LINE__, __func__);
2311                 rte_event_dev_dump(evdev, stdout);
2312                 return -1;
2313         }
2314         /* each packet drop should only be counted in one place - port or dev */
2315         if (stats.rx_dropped != 0) {
2316                 printf("%d:%s: port 1 dropped count not correct\n", __LINE__,
2317                                 __func__);
2318                 rte_event_dev_dump(evdev, stdout);
2319                 return -1;
2320         }
2321
2322         cleanup(t);
2323         return 0;
2324 }
2325
2326 static int
2327 single_packet(struct test *t)
2328 {
2329         const uint32_t MAGIC_SEQN = 7321;
2330         struct rte_event ev;
2331         struct test_event_dev_stats stats;
2332         const int rx_enq = 0;
2333         const int wrk_enq = 2;
2334         int err;
2335
2336         /* Create instance with 4 ports */
2337         if (init(t, 1, 4) < 0 ||
2338                         create_ports(t, 4) < 0 ||
2339                         create_atomic_qids(t, 1) < 0) {
2340                 printf("%d: Error initializing device\n", __LINE__);
2341                 return -1;
2342         }
2343
2344         /* CQ mapping to QID */
2345         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
2346         if (err != 1) {
2347                 printf("%d: error mapping lb qid\n", __LINE__);
2348                 cleanup(t);
2349                 return -1;
2350         }
2351
2352         if (rte_event_dev_start(evdev) < 0) {
2353                 printf("%d: Error with start call\n", __LINE__);
2354                 return -1;
2355         }
2356
2357         /************** Gen pkt and enqueue ****************/
2358         struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2359         if (!arp) {
2360                 printf("%d: gen of pkt failed\n", __LINE__);
2361                 return -1;
2362         }
2363
2364         ev.op = RTE_EVENT_OP_NEW;
2365         ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
2366         ev.mbuf = arp;
2367         ev.queue_id = 0;
2368         ev.flow_id = 3;
2369         arp->seqn = MAGIC_SEQN;
2370
2371         err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2372         if (err != 1) {
2373                 printf("%d: Failed to enqueue\n", __LINE__);
2374                 return -1;
2375         }
2376
2377         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2378
2379         err = test_event_dev_stats_get(evdev, &stats);
2380         if (err) {
2381                 printf("%d: failed to get stats\n", __LINE__);
2382                 return -1;
2383         }
2384
2385         if (stats.rx_pkts != 1 ||
2386                         stats.tx_pkts != 1 ||
2387                         stats.port_inflight[wrk_enq] != 1) {
2388                 printf("%d: Sched core didn't handle pkt as expected\n",
2389                                 __LINE__);
2390                 rte_event_dev_dump(evdev, stdout);
2391                 return -1;
2392         }
2393
2394         uint32_t deq_pkts;
2395
2396         deq_pkts = rte_event_dequeue_burst(evdev, t->port[wrk_enq], &ev, 1, 0);
2397         if (deq_pkts < 1) {
2398                 printf("%d: Failed to deq\n", __LINE__);
2399                 return -1;
2400         }
2401
2402         err = test_event_dev_stats_get(evdev, &stats);
2403         if (err) {
2404                 printf("%d: failed to get stats\n", __LINE__);
2405                 return -1;
2406         }
2407
2408         err = test_event_dev_stats_get(evdev, &stats);
2409         if (ev.mbuf->seqn != MAGIC_SEQN) {
2410                 printf("%d: magic sequence number not dequeued\n", __LINE__);
2411                 return -1;
2412         }
2413
2414         rte_pktmbuf_free(ev.mbuf);
2415         err = rte_event_enqueue_burst(evdev, t->port[wrk_enq], &release_ev, 1);
2416         if (err != 1) {
2417                 printf("%d: Failed to enqueue\n", __LINE__);
2418                 return -1;
2419         }
2420         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2421
2422         err = test_event_dev_stats_get(evdev, &stats);
2423         if (stats.port_inflight[wrk_enq] != 0) {
2424                 printf("%d: port inflight not correct\n", __LINE__);
2425                 return -1;
2426         }
2427
2428         cleanup(t);
2429         return 0;
2430 }
2431
2432 static int
2433 inflight_counts(struct test *t)
2434 {
2435         struct rte_event ev;
2436         struct test_event_dev_stats stats;
2437         const int rx_enq = 0;
2438         const int p1 = 1;
2439         const int p2 = 2;
2440         int err;
2441         int i;
2442
2443         /* Create instance with 4 ports */
2444         if (init(t, 2, 3) < 0 ||
2445                         create_ports(t, 3) < 0 ||
2446                         create_atomic_qids(t, 2) < 0) {
2447                 printf("%d: Error initializing device\n", __LINE__);
2448                 return -1;
2449         }
2450
2451         /* CQ mapping to QID */
2452         err = rte_event_port_link(evdev, t->port[p1], &t->qid[0], NULL, 1);
2453         if (err != 1) {
2454                 printf("%d: error mapping lb qid\n", __LINE__);
2455                 cleanup(t);
2456                 return -1;
2457         }
2458         err = rte_event_port_link(evdev, t->port[p2], &t->qid[1], NULL, 1);
2459         if (err != 1) {
2460                 printf("%d: error mapping lb qid\n", __LINE__);
2461                 cleanup(t);
2462                 return -1;
2463         }
2464
2465         if (rte_event_dev_start(evdev) < 0) {
2466                 printf("%d: Error with start call\n", __LINE__);
2467                 return -1;
2468         }
2469
2470         /************** FORWARD ****************/
2471 #define QID1_NUM 5
2472         for (i = 0; i < QID1_NUM; i++) {
2473                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2474
2475                 if (!arp) {
2476                         printf("%d: gen of pkt failed\n", __LINE__);
2477                         goto err;
2478                 }
2479
2480                 ev.queue_id =  t->qid[0];
2481                 ev.op = RTE_EVENT_OP_NEW;
2482                 ev.mbuf = arp;
2483                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2484                 if (err != 1) {
2485                         printf("%d: Failed to enqueue\n", __LINE__);
2486                         goto err;
2487                 }
2488         }
2489 #define QID2_NUM 3
2490         for (i = 0; i < QID2_NUM; i++) {
2491                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2492
2493                 if (!arp) {
2494                         printf("%d: gen of pkt failed\n", __LINE__);
2495                         goto err;
2496                 }
2497                 ev.queue_id =  t->qid[1];
2498                 ev.op = RTE_EVENT_OP_NEW;
2499                 ev.mbuf = arp;
2500                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2501                 if (err != 1) {
2502                         printf("%d: Failed to enqueue\n", __LINE__);
2503                         goto err;
2504                 }
2505         }
2506
2507         /* schedule */
2508         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2509
2510         err = test_event_dev_stats_get(evdev, &stats);
2511         if (err) {
2512                 printf("%d: failed to get stats\n", __LINE__);
2513                 goto err;
2514         }
2515
2516         if (stats.rx_pkts != QID1_NUM + QID2_NUM ||
2517                         stats.tx_pkts != QID1_NUM + QID2_NUM) {
2518                 printf("%d: Sched core didn't handle pkt as expected\n",
2519                                 __LINE__);
2520                 goto err;
2521         }
2522
2523         if (stats.port_inflight[p1] != QID1_NUM) {
2524                 printf("%d: %s port 1 inflight not correct\n", __LINE__,
2525                                 __func__);
2526                 goto err;
2527         }
2528         if (stats.port_inflight[p2] != QID2_NUM) {
2529                 printf("%d: %s port 2 inflight not correct\n", __LINE__,
2530                                 __func__);
2531                 goto err;
2532         }
2533
2534         /************** DEQUEUE INFLIGHT COUNT CHECKS  ****************/
2535         /* port 1 */
2536         struct rte_event events[QID1_NUM + QID2_NUM];
2537         uint32_t deq_pkts = rte_event_dequeue_burst(evdev, t->port[p1], events,
2538                         RTE_DIM(events), 0);
2539
2540         if (deq_pkts != QID1_NUM) {
2541                 printf("%d: Port 1: DEQUEUE inflight failed\n", __LINE__);
2542                 goto err;
2543         }
2544         err = test_event_dev_stats_get(evdev, &stats);
2545         if (stats.port_inflight[p1] != QID1_NUM) {
2546                 printf("%d: port 1 inflight decrement after DEQ != 0\n",
2547                                 __LINE__);
2548                 goto err;
2549         }
2550         for (i = 0; i < QID1_NUM; i++) {
2551                 err = rte_event_enqueue_burst(evdev, t->port[p1], &release_ev,
2552                                 1);
2553                 if (err != 1) {
2554                         printf("%d: %s rte enqueue of inf release failed\n",
2555                                 __LINE__, __func__);
2556                         goto err;
2557                 }
2558         }
2559
2560         /*
2561          * As the scheduler core decrements inflights, it needs to run to
2562          * process packets to act on the drop messages
2563          */
2564         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2565
2566         err = test_event_dev_stats_get(evdev, &stats);
2567         if (stats.port_inflight[p1] != 0) {
2568                 printf("%d: port 1 inflight NON NULL after DROP\n", __LINE__);
2569                 goto err;
2570         }
2571
2572         /* port2 */
2573         deq_pkts = rte_event_dequeue_burst(evdev, t->port[p2], events,
2574                         RTE_DIM(events), 0);
2575         if (deq_pkts != QID2_NUM) {
2576                 printf("%d: Port 2: DEQUEUE inflight failed\n", __LINE__);
2577                 goto err;
2578         }
2579         err = test_event_dev_stats_get(evdev, &stats);
2580         if (stats.port_inflight[p2] != QID2_NUM) {
2581                 printf("%d: port 1 inflight decrement after DEQ != 0\n",
2582                                 __LINE__);
2583                 goto err;
2584         }
2585         for (i = 0; i < QID2_NUM; i++) {
2586                 err = rte_event_enqueue_burst(evdev, t->port[p2], &release_ev,
2587                                 1);
2588                 if (err != 1) {
2589                         printf("%d: %s rte enqueue of inf release failed\n",
2590                                 __LINE__, __func__);
2591                         goto err;
2592                 }
2593         }
2594
2595         /*
2596          * As the scheduler core decrements inflights, it needs to run to
2597          * process packets to act on the drop messages
2598          */
2599         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2600
2601         err = test_event_dev_stats_get(evdev, &stats);
2602         if (stats.port_inflight[p2] != 0) {
2603                 printf("%d: port 2 inflight NON NULL after DROP\n", __LINE__);
2604                 goto err;
2605         }
2606         cleanup(t);
2607         return 0;
2608
2609 err:
2610         rte_event_dev_dump(evdev, stdout);
2611         cleanup(t);
2612         return -1;
2613 }
2614
2615 static int
2616 parallel_basic(struct test *t, int check_order)
2617 {
2618         const uint8_t rx_port = 0;
2619         const uint8_t w1_port = 1;
2620         const uint8_t w3_port = 3;
2621         const uint8_t tx_port = 4;
2622         int err;
2623         int i;
2624         uint32_t deq_pkts, j;
2625         struct rte_mbuf *mbufs[3];
2626         struct rte_mbuf *mbufs_out[3] = { 0 };
2627         const uint32_t MAGIC_SEQN = 1234;
2628
2629         /* Create instance with 4 ports */
2630         if (init(t, 2, tx_port + 1) < 0 ||
2631                         create_ports(t, tx_port + 1) < 0 ||
2632                         (check_order ?  create_ordered_qids(t, 1) :
2633                                 create_unordered_qids(t, 1)) < 0 ||
2634                         create_directed_qids(t, 1, &tx_port)) {
2635                 printf("%d: Error initializing device\n", __LINE__);
2636                 return -1;
2637         }
2638
2639         /*
2640          * CQ mapping to QID
2641          * We need three ports, all mapped to the same ordered qid0. Then we'll
2642          * take a packet out to each port, re-enqueue in reverse order,
2643          * then make sure the reordering has taken place properly when we
2644          * dequeue from the tx_port.
2645          *
2646          * Simplified test setup diagram:
2647          *
2648          * rx_port        w1_port
2649          *        \     /         \
2650          *         qid0 - w2_port - qid1
2651          *              \         /     \
2652          *                w3_port        tx_port
2653          */
2654         /* CQ mapping to QID for LB ports (directed mapped on create) */
2655         for (i = w1_port; i <= w3_port; i++) {
2656                 err = rte_event_port_link(evdev, t->port[i], &t->qid[0], NULL,
2657                                 1);
2658                 if (err != 1) {
2659                         printf("%d: error mapping lb qid\n", __LINE__);
2660                         cleanup(t);
2661                         return -1;
2662                 }
2663         }
2664
2665         if (rte_event_dev_start(evdev) < 0) {
2666                 printf("%d: Error with start call\n", __LINE__);
2667                 return -1;
2668         }
2669
2670         /* Enqueue 3 packets to the rx port */
2671         for (i = 0; i < 3; i++) {
2672                 struct rte_event ev;
2673                 mbufs[i] = rte_gen_arp(0, t->mbuf_pool);
2674                 if (!mbufs[i]) {
2675                         printf("%d: gen of pkt failed\n", __LINE__);
2676                         return -1;
2677                 }
2678
2679                 ev.queue_id = t->qid[0];
2680                 ev.op = RTE_EVENT_OP_NEW;
2681                 ev.mbuf = mbufs[i];
2682                 mbufs[i]->seqn = MAGIC_SEQN + i;
2683
2684                 /* generate pkt and enqueue */
2685                 err = rte_event_enqueue_burst(evdev, t->port[rx_port], &ev, 1);
2686                 if (err != 1) {
2687                         printf("%d: Failed to enqueue pkt %u, retval = %u\n",
2688                                         __LINE__, i, err);
2689                         return -1;
2690                 }
2691         }
2692
2693         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2694
2695         /* use extra slot to make logic in loops easier */
2696         struct rte_event deq_ev[w3_port + 1];
2697
2698         /* Dequeue the 3 packets, one from each worker port */
2699         for (i = w1_port; i <= w3_port; i++) {
2700                 deq_pkts = rte_event_dequeue_burst(evdev, t->port[i],
2701                                 &deq_ev[i], 1, 0);
2702                 if (deq_pkts != 1) {
2703                         printf("%d: Failed to deq\n", __LINE__);
2704                         rte_event_dev_dump(evdev, stdout);
2705                         return -1;
2706                 }
2707         }
2708
2709         /* Enqueue each packet in reverse order, flushing after each one */
2710         for (i = w3_port; i >= w1_port; i--) {
2711
2712                 deq_ev[i].op = RTE_EVENT_OP_FORWARD;
2713                 deq_ev[i].queue_id = t->qid[1];
2714                 err = rte_event_enqueue_burst(evdev, t->port[i], &deq_ev[i], 1);
2715                 if (err != 1) {
2716                         printf("%d: Failed to enqueue\n", __LINE__);
2717                         return -1;
2718                 }
2719         }
2720         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2721
2722         /* dequeue from the tx ports, we should get 3 packets */
2723         deq_pkts = rte_event_dequeue_burst(evdev, t->port[tx_port], deq_ev,
2724                         3, 0);
2725
2726         /* Check to see if we've got all 3 packets */
2727         if (deq_pkts != 3) {
2728                 printf("%d: expected 3 pkts at tx port got %d from port %d\n",
2729                         __LINE__, deq_pkts, tx_port);
2730                 rte_event_dev_dump(evdev, stdout);
2731                 return 1;
2732         }
2733
2734         /* Check to see if the sequence numbers are in expected order */
2735         if (check_order) {
2736                 for (j = 0 ; j < deq_pkts ; j++) {
2737                         if (deq_ev[j].mbuf->seqn != MAGIC_SEQN + j) {
2738                                 printf(
2739                                         "%d: Incorrect sequence number(%d) from port %d\n",
2740                                         __LINE__, mbufs_out[j]->seqn, tx_port);
2741                                 return -1;
2742                         }
2743                 }
2744         }
2745
2746         /* Destroy the instance */
2747         cleanup(t);
2748         return 0;
2749 }
2750
2751 static int
2752 ordered_basic(struct test *t)
2753 {
2754         return parallel_basic(t, 1);
2755 }
2756
2757 static int
2758 unordered_basic(struct test *t)
2759 {
2760         return parallel_basic(t, 0);
2761 }
2762
2763 static int
2764 holb(struct test *t) /* test to check we avoid basic head-of-line blocking */
2765 {
2766         const struct rte_event new_ev = {
2767                         .op = RTE_EVENT_OP_NEW
2768                         /* all other fields zero */
2769         };
2770         struct rte_event ev = new_ev;
2771         unsigned int rx_port = 0; /* port we get the first flow on */
2772         char rx_port_used_stat[64];
2773         char rx_port_free_stat[64];
2774         char other_port_used_stat[64];
2775
2776         if (init(t, 1, 2) < 0 ||
2777                         create_ports(t, 2) < 0 ||
2778                         create_atomic_qids(t, 1) < 0) {
2779                 printf("%d: Error initializing device\n", __LINE__);
2780                 return -1;
2781         }
2782         int nb_links = rte_event_port_link(evdev, t->port[1], NULL, NULL, 0);
2783         if (rte_event_port_link(evdev, t->port[0], NULL, NULL, 0) != 1 ||
2784                         nb_links != 1) {
2785                 printf("%d: Error links queue to ports\n", __LINE__);
2786                 goto err;
2787         }
2788         if (rte_event_dev_start(evdev) < 0) {
2789                 printf("%d: Error with start call\n", __LINE__);
2790                 goto err;
2791         }
2792
2793         /* send one packet and see where it goes, port 0 or 1 */
2794         if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2795                 printf("%d: Error doing first enqueue\n", __LINE__);
2796                 goto err;
2797         }
2798         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2799
2800         if (rte_event_dev_xstats_by_name_get(evdev, "port_0_cq_ring_used", NULL)
2801                         != 1)
2802                 rx_port = 1;
2803
2804         snprintf(rx_port_used_stat, sizeof(rx_port_used_stat),
2805                         "port_%u_cq_ring_used", rx_port);
2806         snprintf(rx_port_free_stat, sizeof(rx_port_free_stat),
2807                         "port_%u_cq_ring_free", rx_port);
2808         snprintf(other_port_used_stat, sizeof(other_port_used_stat),
2809                         "port_%u_cq_ring_used", rx_port ^ 1);
2810         if (rte_event_dev_xstats_by_name_get(evdev, rx_port_used_stat, NULL)
2811                         != 1) {
2812                 printf("%d: Error, first event not scheduled\n", __LINE__);
2813                 goto err;
2814         }
2815
2816         /* now fill up the rx port's queue with one flow to cause HOLB */
2817         do {
2818                 ev = new_ev;
2819                 if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2820                         printf("%d: Error with enqueue\n", __LINE__);
2821                         goto err;
2822                 }
2823                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
2824         } while (rte_event_dev_xstats_by_name_get(evdev,
2825                                 rx_port_free_stat, NULL) != 0);
2826
2827         /* one more packet, which needs to stay in IQ - i.e. HOLB */
2828         ev = new_ev;
2829         if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2830                 printf("%d: Error with enqueue\n", __LINE__);
2831                 goto err;
2832         }
2833         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2834
2835         /* check that the other port still has an empty CQ */
2836         if (rte_event_dev_xstats_by_name_get(evdev, other_port_used_stat, NULL)
2837                         != 0) {
2838                 printf("%d: Error, second port CQ is not empty\n", __LINE__);
2839                 goto err;
2840         }
2841         /* check IQ now has one packet */
2842         if (rte_event_dev_xstats_by_name_get(evdev, "qid_0_iq_0_used", NULL)
2843                         != 1) {
2844                 printf("%d: Error, QID does not have exactly 1 packet\n",
2845                         __LINE__);
2846                 goto err;
2847         }
2848
2849         /* send another flow, which should pass the other IQ entry */
2850         ev = new_ev;
2851         ev.flow_id = 1;
2852         if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2853                 printf("%d: Error with enqueue\n", __LINE__);
2854                 goto err;
2855         }
2856         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2857
2858         if (rte_event_dev_xstats_by_name_get(evdev, other_port_used_stat, NULL)
2859                         != 1) {
2860                 printf("%d: Error, second flow did not pass out first\n",
2861                         __LINE__);
2862                 goto err;
2863         }
2864
2865         if (rte_event_dev_xstats_by_name_get(evdev, "qid_0_iq_0_used", NULL)
2866                         != 1) {
2867                 printf("%d: Error, QID does not have exactly 1 packet\n",
2868                         __LINE__);
2869                 goto err;
2870         }
2871         cleanup(t);
2872         return 0;
2873 err:
2874         rte_event_dev_dump(evdev, stdout);
2875         cleanup(t);
2876         return -1;
2877 }
2878
2879 static void
2880 flush(uint8_t dev_id __rte_unused, struct rte_event event, void *arg)
2881 {
2882         *((uint8_t *) arg) += (event.u64 == 0xCA11BACC) ? 1 : 0;
2883 }
2884
2885 static int
2886 dev_stop_flush(struct test *t) /* test to check we can properly flush events */
2887 {
2888         const struct rte_event new_ev = {
2889                 .op = RTE_EVENT_OP_NEW,
2890                 .u64 = 0xCA11BACC,
2891                 .queue_id = 0
2892         };
2893         struct rte_event ev = new_ev;
2894         uint8_t count = 0;
2895         int i;
2896
2897         if (init(t, 1, 1) < 0 ||
2898             create_ports(t, 1) < 0 ||
2899             create_atomic_qids(t, 1) < 0) {
2900                 printf("%d: Error initializing device\n", __LINE__);
2901                 return -1;
2902         }
2903
2904         /* Link the queue so *_start() doesn't error out */
2905         if (rte_event_port_link(evdev, t->port[0], NULL, NULL, 0) != 1) {
2906                 printf("%d: Error linking queue to port\n", __LINE__);
2907                 goto err;
2908         }
2909
2910         if (rte_event_dev_start(evdev) < 0) {
2911                 printf("%d: Error with start call\n", __LINE__);
2912                 goto err;
2913         }
2914
2915         for (i = 0; i < DEQUEUE_DEPTH + 1; i++) {
2916                 if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2917                         printf("%d: Error enqueuing events\n", __LINE__);
2918                         goto err;
2919                 }
2920         }
2921
2922         /* Schedule the events from the port to the IQ. At least one event
2923          * should be remaining in the queue.
2924          */
2925         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2926
2927         if (rte_event_dev_stop_flush_callback_register(evdev, flush, &count)) {
2928                 printf("%d: Error installing the flush callback\n", __LINE__);
2929                 goto err;
2930         }
2931
2932         cleanup(t);
2933
2934         if (count == 0) {
2935                 printf("%d: Error executing the flush callback\n", __LINE__);
2936                 goto err;
2937         }
2938
2939         if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
2940                 printf("%d: Error uninstalling the flush callback\n", __LINE__);
2941                 goto err;
2942         }
2943
2944         return 0;
2945 err:
2946         rte_event_dev_dump(evdev, stdout);
2947         cleanup(t);
2948         return -1;
2949 }
2950
2951 static int
2952 worker_loopback_worker_fn(void *arg)
2953 {
2954         struct test *t = arg;
2955         uint8_t port = t->port[1];
2956         int count = 0;
2957         int enqd;
2958
2959         /*
2960          * Takes packets from the input port and then loops them back through
2961          * the Eventdev. Each packet gets looped through QIDs 0-8, 16 times
2962          * so each packet goes through 8*16 = 128 times.
2963          */
2964         printf("%d: \tWorker function started\n", __LINE__);
2965         while (count < NUM_PACKETS) {
2966 #define BURST_SIZE 32
2967                 struct rte_event ev[BURST_SIZE];
2968                 uint16_t i, nb_rx = rte_event_dequeue_burst(evdev, port, ev,
2969                                 BURST_SIZE, 0);
2970                 if (nb_rx == 0) {
2971                         rte_pause();
2972                         continue;
2973                 }
2974
2975                 for (i = 0; i < nb_rx; i++) {
2976                         ev[i].queue_id++;
2977                         if (ev[i].queue_id != 8) {
2978                                 ev[i].op = RTE_EVENT_OP_FORWARD;
2979                                 enqd = rte_event_enqueue_burst(evdev, port,
2980                                                 &ev[i], 1);
2981                                 if (enqd != 1) {
2982                                         printf("%d: Can't enqueue FWD!!\n",
2983                                                         __LINE__);
2984                                         return -1;
2985                                 }
2986                                 continue;
2987                         }
2988
2989                         ev[i].queue_id = 0;
2990                         ev[i].mbuf->udata64++;
2991                         if (ev[i].mbuf->udata64 != 16) {
2992                                 ev[i].op = RTE_EVENT_OP_FORWARD;
2993                                 enqd = rte_event_enqueue_burst(evdev, port,
2994                                                 &ev[i], 1);
2995                                 if (enqd != 1) {
2996                                         printf("%d: Can't enqueue FWD!!\n",
2997                                                         __LINE__);
2998                                         return -1;
2999                                 }
3000                                 continue;
3001                         }
3002                         /* we have hit 16 iterations through system - drop */
3003                         rte_pktmbuf_free(ev[i].mbuf);
3004                         count++;
3005                         ev[i].op = RTE_EVENT_OP_RELEASE;
3006                         enqd = rte_event_enqueue_burst(evdev, port, &ev[i], 1);
3007                         if (enqd != 1) {
3008                                 printf("%d drop enqueue failed\n", __LINE__);
3009                                 return -1;
3010                         }
3011                 }
3012         }
3013
3014         return 0;
3015 }
3016
3017 static int
3018 worker_loopback_producer_fn(void *arg)
3019 {
3020         struct test *t = arg;
3021         uint8_t port = t->port[0];
3022         uint64_t count = 0;
3023
3024         printf("%d: \tProducer function started\n", __LINE__);
3025         while (count < NUM_PACKETS) {
3026                 struct rte_mbuf *m = 0;
3027                 do {
3028                         m = rte_pktmbuf_alloc(t->mbuf_pool);
3029                 } while (m == NULL);
3030
3031                 m->udata64 = 0;
3032
3033                 struct rte_event ev = {
3034                                 .op = RTE_EVENT_OP_NEW,
3035                                 .queue_id = t->qid[0],
3036                                 .flow_id = (uintptr_t)m & 0xFFFF,
3037                                 .mbuf = m,
3038                 };
3039
3040                 if (rte_event_enqueue_burst(evdev, port, &ev, 1) != 1) {
3041                         while (rte_event_enqueue_burst(evdev, port, &ev, 1) !=
3042                                         1)
3043                                 rte_pause();
3044                 }
3045
3046                 count++;
3047         }
3048
3049         return 0;
3050 }
3051
3052 static int
3053 worker_loopback(struct test *t, uint8_t disable_implicit_release)
3054 {
3055         /* use a single producer core, and a worker core to see what happens
3056          * if the worker loops packets back multiple times
3057          */
3058         struct test_event_dev_stats stats;
3059         uint64_t print_cycles = 0, cycles = 0;
3060         uint64_t tx_pkts = 0;
3061         int err;
3062         int w_lcore, p_lcore;
3063
3064         if (init(t, 8, 2) < 0 ||
3065                         create_atomic_qids(t, 8) < 0) {
3066                 printf("%d: Error initializing device\n", __LINE__);
3067                 return -1;
3068         }
3069
3070         /* RX with low max events */
3071         static struct rte_event_port_conf conf = {
3072                         .dequeue_depth = 32,
3073                         .enqueue_depth = 64,
3074         };
3075         /* beware: this cannot be initialized in the static above as it would
3076          * only be initialized once - and this needs to be set for multiple runs
3077          */
3078         conf.new_event_threshold = 512;
3079         conf.event_port_cfg = disable_implicit_release ?
3080                 RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL : 0;
3081
3082         if (rte_event_port_setup(evdev, 0, &conf) < 0) {
3083                 printf("Error setting up RX port\n");
3084                 return -1;
3085         }
3086         t->port[0] = 0;
3087         /* TX with higher max events */
3088         conf.new_event_threshold = 4096;
3089         if (rte_event_port_setup(evdev, 1, &conf) < 0) {
3090                 printf("Error setting up TX port\n");
3091                 return -1;
3092         }
3093         t->port[1] = 1;
3094
3095         /* CQ mapping to QID */
3096         err = rte_event_port_link(evdev, t->port[1], NULL, NULL, 0);
3097         if (err != 8) { /* should have mapped all queues*/
3098                 printf("%d: error mapping port 2 to all qids\n", __LINE__);
3099                 return -1;
3100         }
3101
3102         if (rte_event_dev_start(evdev) < 0) {
3103                 printf("%d: Error with start call\n", __LINE__);
3104                 return -1;
3105         }
3106
3107         p_lcore = rte_get_next_lcore(
3108                         /* start core */ -1,
3109                         /* skip main */ 1,
3110                         /* wrap */ 0);
3111         w_lcore = rte_get_next_lcore(p_lcore, 1, 0);
3112
3113         rte_eal_remote_launch(worker_loopback_producer_fn, t, p_lcore);
3114         rte_eal_remote_launch(worker_loopback_worker_fn, t, w_lcore);
3115
3116         print_cycles = cycles = rte_get_timer_cycles();
3117         while (rte_eal_get_lcore_state(p_lcore) != FINISHED ||
3118                         rte_eal_get_lcore_state(w_lcore) != FINISHED) {
3119
3120                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
3121
3122                 uint64_t new_cycles = rte_get_timer_cycles();
3123
3124                 if (new_cycles - print_cycles > rte_get_timer_hz()) {
3125                         test_event_dev_stats_get(evdev, &stats);
3126                         printf(
3127                                 "%d: \tSched Rx = %"PRIu64", Tx = %"PRIu64"\n",
3128                                 __LINE__, stats.rx_pkts, stats.tx_pkts);
3129
3130                         print_cycles = new_cycles;
3131                 }
3132                 if (new_cycles - cycles > rte_get_timer_hz() * 3) {
3133                         test_event_dev_stats_get(evdev, &stats);
3134                         if (stats.tx_pkts == tx_pkts) {
3135                                 rte_event_dev_dump(evdev, stdout);
3136                                 printf("Dumping xstats:\n");
3137                                 xstats_print();
3138                                 printf(
3139                                         "%d: No schedules for seconds, deadlock\n",
3140                                         __LINE__);
3141                                 return -1;
3142                         }
3143                         tx_pkts = stats.tx_pkts;
3144                         cycles = new_cycles;
3145                 }
3146         }
3147         rte_service_run_iter_on_app_lcore(t->service_id, 1);
3148         /* ensure all completions are flushed */
3149
3150         rte_eal_mp_wait_lcore();
3151
3152         cleanup(t);
3153         return 0;
3154 }
3155
3156 static struct rte_mempool *eventdev_func_mempool;
3157
3158 int
3159 test_sw_eventdev(void)
3160 {
3161         struct test *t;
3162         int ret;
3163
3164         t = malloc(sizeof(struct test));
3165         if (t == NULL)
3166                 return -1;
3167         /* manually initialize the op, older gcc's complain on static
3168          * initialization of struct elements that are a bitfield.
3169          */
3170         release_ev.op = RTE_EVENT_OP_RELEASE;
3171
3172         const char *eventdev_name = "event_sw";
3173         evdev = rte_event_dev_get_dev_id(eventdev_name);
3174         if (evdev < 0) {
3175                 printf("%d: Eventdev %s not found - creating.\n",
3176                                 __LINE__, eventdev_name);
3177                 if (rte_vdev_init(eventdev_name, NULL) < 0) {
3178                         printf("Error creating eventdev\n");
3179                         goto test_fail;
3180                 }
3181                 evdev = rte_event_dev_get_dev_id(eventdev_name);
3182                 if (evdev < 0) {
3183                         printf("Error finding newly created eventdev\n");
3184                         goto test_fail;
3185                 }
3186         }
3187
3188         if (rte_event_dev_service_id_get(evdev, &t->service_id) < 0) {
3189                 printf("Failed to get service ID for software event dev\n");
3190                 goto test_fail;
3191         }
3192
3193         rte_service_runstate_set(t->service_id, 1);
3194         rte_service_set_runstate_mapped_check(t->service_id, 0);
3195
3196         /* Only create mbuf pool once, reuse for each test run */
3197         if (!eventdev_func_mempool) {
3198                 eventdev_func_mempool = rte_pktmbuf_pool_create(
3199                                 "EVENTDEV_SW_SA_MBUF_POOL",
3200                                 (1<<12), /* 4k buffers */
3201                                 32 /*MBUF_CACHE_SIZE*/,
3202                                 0,
3203                                 512, /* use very small mbufs */
3204                                 rte_socket_id());
3205                 if (!eventdev_func_mempool) {
3206                         printf("ERROR creating mempool\n");
3207                         goto test_fail;
3208                 }
3209         }
3210         t->mbuf_pool = eventdev_func_mempool;
3211         printf("*** Running Single Directed Packet test...\n");
3212         ret = test_single_directed_packet(t);
3213         if (ret != 0) {
3214                 printf("ERROR - Single Directed Packet test FAILED.\n");
3215                 goto test_fail;
3216         }
3217         printf("*** Running Directed Forward Credit test...\n");
3218         ret = test_directed_forward_credits(t);
3219         if (ret != 0) {
3220                 printf("ERROR - Directed Forward Credit test FAILED.\n");
3221                 goto test_fail;
3222         }
3223         printf("*** Running Single Load Balanced Packet test...\n");
3224         ret = single_packet(t);
3225         if (ret != 0) {
3226                 printf("ERROR - Single Packet test FAILED.\n");
3227                 goto test_fail;
3228         }
3229         printf("*** Running Unordered Basic test...\n");
3230         ret = unordered_basic(t);
3231         if (ret != 0) {
3232                 printf("ERROR -  Unordered Basic test FAILED.\n");
3233                 goto test_fail;
3234         }
3235         printf("*** Running Ordered Basic test...\n");
3236         ret = ordered_basic(t);
3237         if (ret != 0) {
3238                 printf("ERROR -  Ordered Basic test FAILED.\n");
3239                 goto test_fail;
3240         }
3241         printf("*** Running Burst Packets test...\n");
3242         ret = burst_packets(t);
3243         if (ret != 0) {
3244                 printf("ERROR - Burst Packets test FAILED.\n");
3245                 goto test_fail;
3246         }
3247         printf("*** Running Load Balancing test...\n");
3248         ret = load_balancing(t);
3249         if (ret != 0) {
3250                 printf("ERROR - Load Balancing test FAILED.\n");
3251                 goto test_fail;
3252         }
3253         printf("*** Running Prioritized Directed test...\n");
3254         ret = test_priority_directed(t);
3255         if (ret != 0) {
3256                 printf("ERROR - Prioritized Directed test FAILED.\n");
3257                 goto test_fail;
3258         }
3259         printf("*** Running Prioritized Atomic test...\n");
3260         ret = test_priority_atomic(t);
3261         if (ret != 0) {
3262                 printf("ERROR - Prioritized Atomic test FAILED.\n");
3263                 goto test_fail;
3264         }
3265
3266         printf("*** Running Prioritized Ordered test...\n");
3267         ret = test_priority_ordered(t);
3268         if (ret != 0) {
3269                 printf("ERROR - Prioritized Ordered test FAILED.\n");
3270                 goto test_fail;
3271         }
3272         printf("*** Running Prioritized Unordered test...\n");
3273         ret = test_priority_unordered(t);
3274         if (ret != 0) {
3275                 printf("ERROR - Prioritized Unordered test FAILED.\n");
3276                 goto test_fail;
3277         }
3278         printf("*** Running Invalid QID test...\n");
3279         ret = invalid_qid(t);
3280         if (ret != 0) {
3281                 printf("ERROR - Invalid QID test FAILED.\n");
3282                 goto test_fail;
3283         }
3284         printf("*** Running Load Balancing History test...\n");
3285         ret = load_balancing_history(t);
3286         if (ret != 0) {
3287                 printf("ERROR - Load Balancing History test FAILED.\n");
3288                 goto test_fail;
3289         }
3290         printf("*** Running Inflight Count test...\n");
3291         ret = inflight_counts(t);
3292         if (ret != 0) {
3293                 printf("ERROR - Inflight Count test FAILED.\n");
3294                 goto test_fail;
3295         }
3296         printf("*** Running Abuse Inflights test...\n");
3297         ret = abuse_inflights(t);
3298         if (ret != 0) {
3299                 printf("ERROR - Abuse Inflights test FAILED.\n");
3300                 goto test_fail;
3301         }
3302         printf("*** Running XStats test...\n");
3303         ret = xstats_tests(t);
3304         if (ret != 0) {
3305                 printf("ERROR - XStats test FAILED.\n");
3306                 goto test_fail;
3307         }
3308         printf("*** Running XStats ID Reset test...\n");
3309         ret = xstats_id_reset_tests(t);
3310         if (ret != 0) {
3311                 printf("ERROR - XStats ID Reset test FAILED.\n");
3312                 goto test_fail;
3313         }
3314         printf("*** Running XStats Brute Force test...\n");
3315         ret = xstats_brute_force(t);
3316         if (ret != 0) {
3317                 printf("ERROR - XStats Brute Force test FAILED.\n");
3318                 goto test_fail;
3319         }
3320         printf("*** Running XStats ID Abuse test...\n");
3321         ret = xstats_id_abuse_tests(t);
3322         if (ret != 0) {
3323                 printf("ERROR - XStats ID Abuse test FAILED.\n");
3324                 goto test_fail;
3325         }
3326         printf("*** Running QID Priority test...\n");
3327         ret = qid_priorities(t);
3328         if (ret != 0) {
3329                 printf("ERROR - QID Priority test FAILED.\n");
3330                 goto test_fail;
3331         }
3332         printf("*** Running Unlink-in-progress test...\n");
3333         ret = unlink_in_progress(t);
3334         if (ret != 0) {
3335                 printf("ERROR - Unlink in progress test FAILED.\n");
3336                 goto test_fail;
3337         }
3338         printf("*** Running Ordered Reconfigure test...\n");
3339         ret = ordered_reconfigure(t);
3340         if (ret != 0) {
3341                 printf("ERROR - Ordered Reconfigure test FAILED.\n");
3342                 goto test_fail;
3343         }
3344         printf("*** Running Port LB Single Reconfig test...\n");
3345         ret = port_single_lb_reconfig(t);
3346         if (ret != 0) {
3347                 printf("ERROR - Port LB Single Reconfig test FAILED.\n");
3348                 goto test_fail;
3349         }
3350         printf("*** Running Port Reconfig Credits test...\n");
3351         ret = port_reconfig_credits(t);
3352         if (ret != 0) {
3353                 printf("ERROR - Port Reconfig Credits Reset test FAILED.\n");
3354                 goto test_fail;
3355         }
3356         printf("*** Running Head-of-line-blocking test...\n");
3357         ret = holb(t);
3358         if (ret != 0) {
3359                 printf("ERROR - Head-of-line-blocking test FAILED.\n");
3360                 goto test_fail;
3361         }
3362         printf("*** Running Stop Flush test...\n");
3363         ret = dev_stop_flush(t);
3364         if (ret != 0) {
3365                 printf("ERROR - Stop Flush test FAILED.\n");
3366                 goto test_fail;
3367         }
3368         if (rte_lcore_count() >= 3) {
3369                 printf("*** Running Worker loopback test...\n");
3370                 ret = worker_loopback(t, 0);
3371                 if (ret != 0) {
3372                         printf("ERROR - Worker loopback test FAILED.\n");
3373                         return ret;
3374                 }
3375
3376                 printf("*** Running Worker loopback test (implicit release disabled)...\n");
3377                 ret = worker_loopback(t, 1);
3378                 if (ret != 0) {
3379                         printf("ERROR - Worker loopback test FAILED.\n");
3380                         goto test_fail;
3381                 }
3382         } else {
3383                 printf("### Not enough cores for worker loopback tests.\n");
3384                 printf("### Need at least 3 cores for the tests.\n");
3385         }
3386
3387         /*
3388          * Free test instance, leaving mempool initialized, and a pointer to it
3389          * in static eventdev_func_mempool, as it is re-used on re-runs
3390          */
3391         free(t);
3392
3393         printf("SW Eventdev Selftest Successful.\n");
3394         return 0;
3395 test_fail:
3396         free(t);
3397         printf("SW Eventdev Selftest Failed.\n");
3398         return -1;
3399 }