drivers/event/sw/sw_evdev_selftest.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <string.h>
   7 #include <stdint.h>
   8 #include <errno.h>
   9 #include <unistd.h>
  10 #include <sys/queue.h>
  11
  12 #include <rte_memory.h>
  13 #include <rte_launch.h>
  14 #include <rte_eal.h>
  15 #include <rte_per_lcore.h>
  16 #include <rte_lcore.h>
  17 #include <rte_debug.h>
  18 #include <rte_ethdev.h>
  19 #include <rte_cycles.h>
  20 #include <rte_eventdev.h>
  21 #include <rte_pause.h>
  22 #include <rte_service.h>
  23 #include <rte_service_component.h>
  24 #include <rte_bus_vdev.h>
  25
  26 #include "sw_evdev.h"
  27
  28 #define MAX_PORTS 16
  29 #define MAX_QIDS 16
  30 #define NUM_PACKETS (1<<18)
  31 #define DEQUEUE_DEPTH 128
  32
  33 static int evdev;
  34
  35 struct test {
  36         struct rte_mempool *mbuf_pool;
  37         uint8_t port[MAX_PORTS];
  38         uint8_t qid[MAX_QIDS];
  39         int nb_qids;
  40         uint32_t service_id;
  41 };
  42
  43 typedef uint8_t counter_dynfield_t;
  44 static int counter_dynfield_offset = -1;
  45
  46 static inline counter_dynfield_t *
  47 counter_field(struct rte_mbuf *mbuf)
  48 {
  49         return RTE_MBUF_DYNFIELD(mbuf, \
  50                         counter_dynfield_offset, counter_dynfield_t *);
  51 }
  52
  53 static struct rte_event release_ev;
  54
  55 static inline struct rte_mbuf *
  56 rte_gen_arp(int portid, struct rte_mempool *mp)
  57 {
  58         /*
  59          * len = 14 + 46
  60          * ARP, Request who-has 10.0.0.1 tell 10.0.0.2, length 46
  61          */
  62         static const uint8_t arp_request[] = {
  63                 /*0x0000:*/ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xec, 0xa8,
  64                 0x6b, 0xfd, 0x02, 0x29, 0x08, 0x06, 0x00, 0x01,
  65                 /*0x0010:*/ 0x08, 0x00, 0x06, 0x04, 0x00, 0x01, 0xec, 0xa8,
  66                 0x6b, 0xfd, 0x02, 0x29, 0x0a, 0x00, 0x00, 0x01,
  67                 /*0x0020:*/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00,
  68                 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  69                 /*0x0030:*/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  70                 0x00, 0x00, 0x00, 0x00
  71         };
  72         struct rte_mbuf *m;
  73         int pkt_len = sizeof(arp_request) - 1;
  74
  75         m = rte_pktmbuf_alloc(mp);
  76         if (!m)
  77                 return 0;
  78
  79         memcpy((void *)((uintptr_t)m->buf_addr + m->data_off),
  80                 arp_request, pkt_len);
  81         rte_pktmbuf_pkt_len(m) = pkt_len;
  82         rte_pktmbuf_data_len(m) = pkt_len;
  83
  84         RTE_SET_USED(portid);
  85
  86         return m;
  87 }
  88
  89 static void
  90 xstats_print(void)
  91 {
  92         const uint32_t XSTATS_MAX = 1024;
  93         uint32_t i;
  94         uint32_t ids[XSTATS_MAX];
  95         uint64_t values[XSTATS_MAX];
  96         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
  97
  98         for (i = 0; i < XSTATS_MAX; i++)
  99                 ids[i] = i;
 100
 101         /* Device names / values */
 102         int ret = rte_event_dev_xstats_names_get(evdev,
 103                                         RTE_EVENT_DEV_XSTATS_DEVICE, 0,
 104                                         xstats_names, ids, XSTATS_MAX);
 105         if (ret < 0) {
 106                 printf("%d: xstats names get() returned error\n",
 107                         __LINE__);
 108                 return;
 109         }
 110         ret = rte_event_dev_xstats_get(evdev,
 111                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 112                                         0, ids, values, ret);
 113         if (ret > (signed int)XSTATS_MAX)
 114                 printf("%s %d: more xstats available than space\n",
 115                                 __func__, __LINE__);
 116         for (i = 0; (signed int)i < ret; i++) {
 117                 printf("%d : %s : %"PRIu64"\n",
 118                                 i, xstats_names[i].name, values[i]);
 119         }
 120
 121         /* Port names / values */
 122         ret = rte_event_dev_xstats_names_get(evdev,
 123                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
 124                                         xstats_names, ids, XSTATS_MAX);
 125         ret = rte_event_dev_xstats_get(evdev,
 126                                         RTE_EVENT_DEV_XSTATS_PORT, 1,
 127                                         ids, values, ret);
 128         if (ret > (signed int)XSTATS_MAX)
 129                 printf("%s %d: more xstats available than space\n",
 130                                 __func__, __LINE__);
 131         for (i = 0; (signed int)i < ret; i++) {
 132                 printf("%d : %s : %"PRIu64"\n",
 133                                 i, xstats_names[i].name, values[i]);
 134         }
 135
 136         /* Queue names / values */
 137         ret = rte_event_dev_xstats_names_get(evdev,
 138                                         RTE_EVENT_DEV_XSTATS_QUEUE, 0,
 139                                         xstats_names, ids, XSTATS_MAX);
 140         ret = rte_event_dev_xstats_get(evdev,
 141                                         RTE_EVENT_DEV_XSTATS_QUEUE,
 142                                         1, ids, values, ret);
 143         if (ret > (signed int)XSTATS_MAX)
 144                 printf("%s %d: more xstats available than space\n",
 145                                 __func__, __LINE__);
 146         for (i = 0; (signed int)i < ret; i++) {
 147                 printf("%d : %s : %"PRIu64"\n",
 148                                 i, xstats_names[i].name, values[i]);
 149         }
 150 }
 151
 152 /* initialization and config */
 153 static inline int
 154 init(struct test *t, int nb_queues, int nb_ports)
 155 {
 156         struct rte_event_dev_config config = {
 157                         .nb_event_queues = nb_queues,
 158                         .nb_event_ports = nb_ports,
 159                         .nb_event_queue_flows = 1024,
 160                         .nb_events_limit = 4096,
 161                         .nb_event_port_dequeue_depth = DEQUEUE_DEPTH,
 162                         .nb_event_port_enqueue_depth = 128,
 163         };
 164         int ret;
 165
 166         void *temp = t->mbuf_pool; /* save and restore mbuf pool */
 167
 168         memset(t, 0, sizeof(*t));
 169         t->mbuf_pool = temp;
 170
 171         ret = rte_event_dev_configure(evdev, &config);
 172         if (ret < 0)
 173                 printf("%d: Error configuring device\n", __LINE__);
 174         return ret;
 175 };
 176
 177 static inline int
 178 create_ports(struct test *t, int num_ports)
 179 {
 180         int i;
 181         static const struct rte_event_port_conf conf = {
 182                         .new_event_threshold = 1024,
 183                         .dequeue_depth = 32,
 184                         .enqueue_depth = 64,
 185         };
 186         if (num_ports > MAX_PORTS)
 187                 return -1;
 188
 189         for (i = 0; i < num_ports; i++) {
 190                 if (rte_event_port_setup(evdev, i, &conf) < 0) {
 191                         printf("Error setting up port %d\n", i);
 192                         return -1;
 193                 }
 194                 t->port[i] = i;
 195         }
 196
 197         return 0;
 198 }
 199
 200 static inline int
 201 create_lb_qids(struct test *t, int num_qids, uint32_t flags)
 202 {
 203         int i;
 204
 205         /* Q creation */
 206         const struct rte_event_queue_conf conf = {
 207                         .schedule_type = flags,
 208                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 209                         .nb_atomic_flows = 1024,
 210                         .nb_atomic_order_sequences = 1024,
 211         };
 212
 213         for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
 214                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
 215                         printf("%d: error creating qid %d\n", __LINE__, i);
 216                         return -1;
 217                 }
 218                 t->qid[i] = i;
 219         }
 220         t->nb_qids += num_qids;
 221         if (t->nb_qids > MAX_QIDS)
 222                 return -1;
 223
 224         return 0;
 225 }
 226
 227 static inline int
 228 create_atomic_qids(struct test *t, int num_qids)
 229 {
 230         return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
 231 }
 232
 233 static inline int
 234 create_ordered_qids(struct test *t, int num_qids)
 235 {
 236         return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ORDERED);
 237 }
 238
 239
 240 static inline int
 241 create_unordered_qids(struct test *t, int num_qids)
 242 {
 243         return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_PARALLEL);
 244 }
 245
 246 static inline int
 247 create_directed_qids(struct test *t, int num_qids, const uint8_t ports[])
 248 {
 249         int i;
 250
 251         /* Q creation */
 252         static const struct rte_event_queue_conf conf = {
 253                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 254                         .event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK,
 255         };
 256
 257         for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
 258                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
 259                         printf("%d: error creating qid %d\n", __LINE__, i);
 260                         return -1;
 261                 }
 262                 t->qid[i] = i;
 263
 264                 if (rte_event_port_link(evdev, ports[i - t->nb_qids],
 265                                 &t->qid[i], NULL, 1) != 1) {
 266                         printf("%d: error creating link for qid %d\n",
 267                                         __LINE__, i);
 268                         return -1;
 269                 }
 270         }
 271         t->nb_qids += num_qids;
 272         if (t->nb_qids > MAX_QIDS)
 273                 return -1;
 274
 275         return 0;
 276 }
 277
 278 /* destruction */
 279 static inline int
 280 cleanup(struct test *t __rte_unused)
 281 {
 282         rte_event_dev_stop(evdev);
 283         rte_event_dev_close(evdev);
 284         return 0;
 285 };
 286
 287 struct test_event_dev_stats {
 288         uint64_t rx_pkts;       /**< Total packets received */
 289         uint64_t rx_dropped;    /**< Total packets dropped (Eg Invalid QID) */
 290         uint64_t tx_pkts;       /**< Total packets transmitted */
 291
 292         /** Packets received on this port */
 293         uint64_t port_rx_pkts[MAX_PORTS];
 294         /** Packets dropped on this port */
 295         uint64_t port_rx_dropped[MAX_PORTS];
 296         /** Packets inflight on this port */
 297         uint64_t port_inflight[MAX_PORTS];
 298         /** Packets transmitted on this port */
 299         uint64_t port_tx_pkts[MAX_PORTS];
 300         /** Packets received on this qid */
 301         uint64_t qid_rx_pkts[MAX_QIDS];
 302         /** Packets dropped on this qid */
 303         uint64_t qid_rx_dropped[MAX_QIDS];
 304         /** Packets transmitted on this qid */
 305         uint64_t qid_tx_pkts[MAX_QIDS];
 306 };
 307
 308 static inline int
 309 test_event_dev_stats_get(int dev_id, struct test_event_dev_stats *stats)
 310 {
 311         static uint32_t i;
 312         static uint32_t total_ids[3]; /* rx, tx and drop */
 313         static uint32_t port_rx_pkts_ids[MAX_PORTS];
 314         static uint32_t port_rx_dropped_ids[MAX_PORTS];
 315         static uint32_t port_inflight_ids[MAX_PORTS];
 316         static uint32_t port_tx_pkts_ids[MAX_PORTS];
 317         static uint32_t qid_rx_pkts_ids[MAX_QIDS];
 318         static uint32_t qid_rx_dropped_ids[MAX_QIDS];
 319         static uint32_t qid_tx_pkts_ids[MAX_QIDS];
 320
 321
 322         stats->rx_pkts = rte_event_dev_xstats_by_name_get(dev_id,
 323                         "dev_rx", &total_ids[0]);
 324         stats->rx_dropped = rte_event_dev_xstats_by_name_get(dev_id,
 325                         "dev_drop", &total_ids[1]);
 326         stats->tx_pkts = rte_event_dev_xstats_by_name_get(dev_id,
 327                         "dev_tx", &total_ids[2]);
 328         for (i = 0; i < MAX_PORTS; i++) {
 329                 char name[32];
 330                 snprintf(name, sizeof(name), "port_%u_rx", i);
 331                 stats->port_rx_pkts[i] = rte_event_dev_xstats_by_name_get(
 332                                 dev_id, name, &port_rx_pkts_ids[i]);
 333                 snprintf(name, sizeof(name), "port_%u_drop", i);
 334                 stats->port_rx_dropped[i] = rte_event_dev_xstats_by_name_get(
 335                                 dev_id, name, &port_rx_dropped_ids[i]);
 336                 snprintf(name, sizeof(name), "port_%u_inflight", i);
 337                 stats->port_inflight[i] = rte_event_dev_xstats_by_name_get(
 338                                 dev_id, name, &port_inflight_ids[i]);
 339                 snprintf(name, sizeof(name), "port_%u_tx", i);
 340                 stats->port_tx_pkts[i] = rte_event_dev_xstats_by_name_get(
 341                                 dev_id, name, &port_tx_pkts_ids[i]);
 342         }
 343         for (i = 0; i < MAX_QIDS; i++) {
 344                 char name[32];
 345                 snprintf(name, sizeof(name), "qid_%u_rx", i);
 346                 stats->qid_rx_pkts[i] = rte_event_dev_xstats_by_name_get(
 347                                 dev_id, name, &qid_rx_pkts_ids[i]);
 348                 snprintf(name, sizeof(name), "qid_%u_drop", i);
 349                 stats->qid_rx_dropped[i] = rte_event_dev_xstats_by_name_get(
 350                                 dev_id, name, &qid_rx_dropped_ids[i]);
 351                 snprintf(name, sizeof(name), "qid_%u_tx", i);
 352                 stats->qid_tx_pkts[i] = rte_event_dev_xstats_by_name_get(
 353                                 dev_id, name, &qid_tx_pkts_ids[i]);
 354         }
 355
 356         return 0;
 357 }
 358
 359 /* run_prio_packet_test
 360  * This performs a basic packet priority check on the test instance passed in.
 361  * It is factored out of the main priority tests as the same tests must be
 362  * performed to ensure prioritization of each type of QID.
 363  *
 364  * Requirements:
 365  *  - An initialized test structure, including mempool
 366  *  - t->port[0] is initialized for both Enq / Deq of packets to the QID
 367  *  - t->qid[0] is the QID to be tested
 368  *  - if LB QID, the CQ must be mapped to the QID.
 369  */
 370 static int
 371 run_prio_packet_test(struct test *t)
 372 {
 373         int err;
 374         const uint32_t MAGIC_SEQN[] = {4711, 1234};
 375         const uint32_t PRIORITY[] = {
 376                 RTE_EVENT_DEV_PRIORITY_NORMAL,
 377                 RTE_EVENT_DEV_PRIORITY_HIGHEST
 378         };
 379         unsigned int i;
 380         for (i = 0; i < RTE_DIM(MAGIC_SEQN); i++) {
 381                 /* generate pkt and enqueue */
 382                 struct rte_event ev;
 383                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
 384                 if (!arp) {
 385                         printf("%d: gen of pkt failed\n", __LINE__);
 386                         return -1;
 387                 }
 388                 arp->seqn = MAGIC_SEQN[i];
 389
 390                 ev = (struct rte_event){
 391                         .priority = PRIORITY[i],
 392                         .op = RTE_EVENT_OP_NEW,
 393                         .queue_id = t->qid[0],
 394                         .mbuf = arp
 395                 };
 396                 err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
 397                 if (err != 1) {
 398                         printf("%d: error failed to enqueue\n", __LINE__);
 399                         return -1;
 400                 }
 401         }
 402
 403         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 404
 405         struct test_event_dev_stats stats;
 406         err = test_event_dev_stats_get(evdev, &stats);
 407         if (err) {
 408                 printf("%d: error failed to get stats\n", __LINE__);
 409                 return -1;
 410         }
 411
 412         if (stats.port_rx_pkts[t->port[0]] != 2) {
 413                 printf("%d: error stats incorrect for directed port\n",
 414                                 __LINE__);
 415                 rte_event_dev_dump(evdev, stdout);
 416                 return -1;
 417         }
 418
 419         struct rte_event ev, ev2;
 420         uint32_t deq_pkts;
 421         deq_pkts = rte_event_dequeue_burst(evdev, t->port[0], &ev, 1, 0);
 422         if (deq_pkts != 1) {
 423                 printf("%d: error failed to deq\n", __LINE__);
 424                 rte_event_dev_dump(evdev, stdout);
 425                 return -1;
 426         }
 427         if (ev.mbuf->seqn != MAGIC_SEQN[1]) {
 428                 printf("%d: first packet out not highest priority\n",
 429                                 __LINE__);
 430                 rte_event_dev_dump(evdev, stdout);
 431                 return -1;
 432         }
 433         rte_pktmbuf_free(ev.mbuf);
 434
 435         deq_pkts = rte_event_dequeue_burst(evdev, t->port[0], &ev2, 1, 0);
 436         if (deq_pkts != 1) {
 437                 printf("%d: error failed to deq\n", __LINE__);
 438                 rte_event_dev_dump(evdev, stdout);
 439                 return -1;
 440         }
 441         if (ev2.mbuf->seqn != MAGIC_SEQN[0]) {
 442                 printf("%d: second packet out not lower priority\n",
 443                                 __LINE__);
 444                 rte_event_dev_dump(evdev, stdout);
 445                 return -1;
 446         }
 447         rte_pktmbuf_free(ev2.mbuf);
 448
 449         cleanup(t);
 450         return 0;
 451 }
 452
 453 static int
 454 test_single_directed_packet(struct test *t)
 455 {
 456         const int rx_enq = 0;
 457         const int wrk_enq = 2;
 458         int err;
 459
 460         /* Create instance with 3 directed QIDs going to 3 ports */
 461         if (init(t, 3, 3) < 0 ||
 462                         create_ports(t, 3) < 0 ||
 463                         create_directed_qids(t, 3, t->port) < 0)
 464                 return -1;
 465
 466         if (rte_event_dev_start(evdev) < 0) {
 467                 printf("%d: Error with start call\n", __LINE__);
 468                 return -1;
 469         }
 470
 471         /************** FORWARD ****************/
 472         struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
 473         struct rte_event ev = {
 474                         .op = RTE_EVENT_OP_NEW,
 475                         .queue_id = wrk_enq,
 476                         .mbuf = arp,
 477         };
 478
 479         if (!arp) {
 480                 printf("%d: gen of pkt failed\n", __LINE__);
 481                 return -1;
 482         }
 483
 484         const uint32_t MAGIC_SEQN = 4711;
 485         arp->seqn = MAGIC_SEQN;
 486
 487         /* generate pkt and enqueue */
 488         err = rte_event_enqueue_burst(evdev, rx_enq, &ev, 1);
 489         if (err != 1) {
 490                 printf("%d: error failed to enqueue\n", __LINE__);
 491                 return -1;
 492         }
 493
 494         /* Run schedule() as dir packets may need to be re-ordered */
 495         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 496
 497         struct test_event_dev_stats stats;
 498         err = test_event_dev_stats_get(evdev, &stats);
 499         if (err) {
 500                 printf("%d: error failed to get stats\n", __LINE__);
 501                 return -1;
 502         }
 503
 504         if (stats.port_rx_pkts[rx_enq] != 1) {
 505                 printf("%d: error stats incorrect for directed port\n",
 506                                 __LINE__);
 507                 return -1;
 508         }
 509
 510         uint32_t deq_pkts;
 511         deq_pkts = rte_event_dequeue_burst(evdev, wrk_enq, &ev, 1, 0);
 512         if (deq_pkts != 1) {
 513                 printf("%d: error failed to deq\n", __LINE__);
 514                 return -1;
 515         }
 516
 517         err = test_event_dev_stats_get(evdev, &stats);
 518         if (stats.port_rx_pkts[wrk_enq] != 0 &&
 519                         stats.port_rx_pkts[wrk_enq] != 1) {
 520                 printf("%d: error directed stats post-dequeue\n", __LINE__);
 521                 return -1;
 522         }
 523
 524         if (ev.mbuf->seqn != MAGIC_SEQN) {
 525                 printf("%d: error magic sequence number not dequeued\n",
 526                                 __LINE__);
 527                 return -1;
 528         }
 529
 530         rte_pktmbuf_free(ev.mbuf);
 531         cleanup(t);
 532         return 0;
 533 }
 534
 535 static int
 536 test_directed_forward_credits(struct test *t)
 537 {
 538         uint32_t i;
 539         int32_t err;
 540
 541         if (init(t, 1, 1) < 0 ||
 542                         create_ports(t, 1) < 0 ||
 543                         create_directed_qids(t, 1, t->port) < 0)
 544                 return -1;
 545
 546         if (rte_event_dev_start(evdev) < 0) {
 547                 printf("%d: Error with start call\n", __LINE__);
 548                 return -1;
 549         }
 550
 551         struct rte_event ev = {
 552                         .op = RTE_EVENT_OP_NEW,
 553                         .queue_id = 0,
 554         };
 555
 556         for (i = 0; i < 1000; i++) {
 557                 err = rte_event_enqueue_burst(evdev, 0, &ev, 1);
 558                 if (err != 1) {
 559                         printf("%d: error failed to enqueue\n", __LINE__);
 560                         return -1;
 561                 }
 562                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
 563
 564                 uint32_t deq_pkts;
 565                 deq_pkts = rte_event_dequeue_burst(evdev, 0, &ev, 1, 0);
 566                 if (deq_pkts != 1) {
 567                         printf("%d: error failed to deq\n", __LINE__);
 568                         return -1;
 569                 }
 570
 571                 /* re-write event to be a forward, and continue looping it */
 572                 ev.op = RTE_EVENT_OP_FORWARD;
 573         }
 574
 575         cleanup(t);
 576         return 0;
 577 }
 578
 579
 580 static int
 581 test_priority_directed(struct test *t)
 582 {
 583         if (init(t, 1, 1) < 0 ||
 584                         create_ports(t, 1) < 0 ||
 585                         create_directed_qids(t, 1, t->port) < 0) {
 586                 printf("%d: Error initializing device\n", __LINE__);
 587                 return -1;
 588         }
 589
 590         if (rte_event_dev_start(evdev) < 0) {
 591                 printf("%d: Error with start call\n", __LINE__);
 592                 return -1;
 593         }
 594
 595         return run_prio_packet_test(t);
 596 }
 597
 598 static int
 599 test_priority_atomic(struct test *t)
 600 {
 601         if (init(t, 1, 1) < 0 ||
 602                         create_ports(t, 1) < 0 ||
 603                         create_atomic_qids(t, 1) < 0) {
 604                 printf("%d: Error initializing device\n", __LINE__);
 605                 return -1;
 606         }
 607
 608         /* map the QID */
 609         if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) {
 610                 printf("%d: error mapping qid to port\n", __LINE__);
 611                 return -1;
 612         }
 613         if (rte_event_dev_start(evdev) < 0) {
 614                 printf("%d: Error with start call\n", __LINE__);
 615                 return -1;
 616         }
 617
 618         return run_prio_packet_test(t);
 619 }
 620
 621 static int
 622 test_priority_ordered(struct test *t)
 623 {
 624         if (init(t, 1, 1) < 0 ||
 625                         create_ports(t, 1) < 0 ||
 626                         create_ordered_qids(t, 1) < 0) {
 627                 printf("%d: Error initializing device\n", __LINE__);
 628                 return -1;
 629         }
 630
 631         /* map the QID */
 632         if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) {
 633                 printf("%d: error mapping qid to port\n", __LINE__);
 634                 return -1;
 635         }
 636         if (rte_event_dev_start(evdev) < 0) {
 637                 printf("%d: Error with start call\n", __LINE__);
 638                 return -1;
 639         }
 640
 641         return run_prio_packet_test(t);
 642 }
 643
 644 static int
 645 test_priority_unordered(struct test *t)
 646 {
 647         if (init(t, 1, 1) < 0 ||
 648                         create_ports(t, 1) < 0 ||
 649                         create_unordered_qids(t, 1) < 0) {
 650                 printf("%d: Error initializing device\n", __LINE__);
 651                 return -1;
 652         }
 653
 654         /* map the QID */
 655         if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) {
 656                 printf("%d: error mapping qid to port\n", __LINE__);
 657                 return -1;
 658         }
 659         if (rte_event_dev_start(evdev) < 0) {
 660                 printf("%d: Error with start call\n", __LINE__);
 661                 return -1;
 662         }
 663
 664         return run_prio_packet_test(t);
 665 }
 666
 667 static int
 668 burst_packets(struct test *t)
 669 {
 670         /************** CONFIG ****************/
 671         uint32_t i;
 672         int err;
 673         int ret;
 674
 675         /* Create instance with 2 ports and 2 queues */
 676         if (init(t, 2, 2) < 0 ||
 677                         create_ports(t, 2) < 0 ||
 678                         create_atomic_qids(t, 2) < 0) {
 679                 printf("%d: Error initializing device\n", __LINE__);
 680                 return -1;
 681         }
 682
 683         /* CQ mapping to QID */
 684         ret = rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1);
 685         if (ret != 1) {
 686                 printf("%d: error mapping lb qid0\n", __LINE__);
 687                 return -1;
 688         }
 689         ret = rte_event_port_link(evdev, t->port[1], &t->qid[1], NULL, 1);
 690         if (ret != 1) {
 691                 printf("%d: error mapping lb qid1\n", __LINE__);
 692                 return -1;
 693         }
 694
 695         if (rte_event_dev_start(evdev) < 0) {
 696                 printf("%d: Error with start call\n", __LINE__);
 697                 return -1;
 698         }
 699
 700         /************** FORWARD ****************/
 701         const uint32_t rx_port = 0;
 702         const uint32_t NUM_PKTS = 2;
 703
 704         for (i = 0; i < NUM_PKTS; i++) {
 705                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
 706                 if (!arp) {
 707                         printf("%d: error generating pkt\n", __LINE__);
 708                         return -1;
 709                 }
 710
 711                 struct rte_event ev = {
 712                                 .op = RTE_EVENT_OP_NEW,
 713                                 .queue_id = i % 2,
 714                                 .flow_id = i % 3,
 715                                 .mbuf = arp,
 716                 };
 717                 /* generate pkt and enqueue */
 718                 err = rte_event_enqueue_burst(evdev, t->port[rx_port], &ev, 1);
 719                 if (err != 1) {
 720                         printf("%d: Failed to enqueue\n", __LINE__);
 721                         return -1;
 722                 }
 723         }
 724         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 725
 726         /* Check stats for all NUM_PKTS arrived to sched core */
 727         struct test_event_dev_stats stats;
 728
 729         err = test_event_dev_stats_get(evdev, &stats);
 730         if (err) {
 731                 printf("%d: failed to get stats\n", __LINE__);
 732                 return -1;
 733         }
 734         if (stats.rx_pkts != NUM_PKTS || stats.tx_pkts != NUM_PKTS) {
 735                 printf("%d: Sched core didn't receive all %d pkts\n",
 736                                 __LINE__, NUM_PKTS);
 737                 rte_event_dev_dump(evdev, stdout);
 738                 return -1;
 739         }
 740
 741         uint32_t deq_pkts;
 742         int p;
 743
 744         deq_pkts = 0;
 745         /******** DEQ QID 1 *******/
 746         do {
 747                 struct rte_event ev;
 748                 p = rte_event_dequeue_burst(evdev, t->port[0], &ev, 1, 0);
 749                 deq_pkts += p;
 750                 rte_pktmbuf_free(ev.mbuf);
 751         } while (p);
 752
 753         if (deq_pkts != NUM_PKTS/2) {
 754                 printf("%d: Half of NUM_PKTS didn't arrive at port 1\n",
 755                                 __LINE__);
 756                 return -1;
 757         }
 758
 759         /******** DEQ QID 2 *******/
 760         deq_pkts = 0;
 761         do {
 762                 struct rte_event ev;
 763                 p = rte_event_dequeue_burst(evdev, t->port[1], &ev, 1, 0);
 764                 deq_pkts += p;
 765                 rte_pktmbuf_free(ev.mbuf);
 766         } while (p);
 767         if (deq_pkts != NUM_PKTS/2) {
 768                 printf("%d: Half of NUM_PKTS didn't arrive at port 2\n",
 769                                 __LINE__);
 770                 return -1;
 771         }
 772
 773         cleanup(t);
 774         return 0;
 775 }
 776
 777 static int
 778 abuse_inflights(struct test *t)
 779 {
 780         const int rx_enq = 0;
 781         const int wrk_enq = 2;
 782         int err;
 783
 784         /* Create instance with 4 ports */
 785         if (init(t, 1, 4) < 0 ||
 786                         create_ports(t, 4) < 0 ||
 787                         create_atomic_qids(t, 1) < 0) {
 788                 printf("%d: Error initializing device\n", __LINE__);
 789                 return -1;
 790         }
 791
 792         /* CQ mapping to QID */
 793         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
 794         if (err != 1) {
 795                 printf("%d: error mapping lb qid\n", __LINE__);
 796                 cleanup(t);
 797                 return -1;
 798         }
 799
 800         if (rte_event_dev_start(evdev) < 0) {
 801                 printf("%d: Error with start call\n", __LINE__);
 802                 return -1;
 803         }
 804
 805         /* Enqueue op only */
 806         err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &release_ev, 1);
 807         if (err != 1) {
 808                 printf("%d: Failed to enqueue\n", __LINE__);
 809                 return -1;
 810         }
 811
 812         /* schedule */
 813         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 814
 815         struct test_event_dev_stats stats;
 816
 817         err = test_event_dev_stats_get(evdev, &stats);
 818         if (err) {
 819                 printf("%d: failed to get stats\n", __LINE__);
 820                 return -1;
 821         }
 822
 823         if (stats.rx_pkts != 0 ||
 824                         stats.tx_pkts != 0 ||
 825                         stats.port_inflight[wrk_enq] != 0) {
 826                 printf("%d: Sched core didn't handle pkt as expected\n",
 827                                 __LINE__);
 828                 return -1;
 829         }
 830
 831         cleanup(t);
 832         return 0;
 833 }
 834
 835 static int
 836 xstats_tests(struct test *t)
 837 {
 838         const int wrk_enq = 2;
 839         int err;
 840
 841         /* Create instance with 4 ports */
 842         if (init(t, 1, 4) < 0 ||
 843                         create_ports(t, 4) < 0 ||
 844                         create_atomic_qids(t, 1) < 0) {
 845                 printf("%d: Error initializing device\n", __LINE__);
 846                 return -1;
 847         }
 848
 849         /* CQ mapping to QID */
 850         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
 851         if (err != 1) {
 852                 printf("%d: error mapping lb qid\n", __LINE__);
 853                 cleanup(t);
 854                 return -1;
 855         }
 856
 857         if (rte_event_dev_start(evdev) < 0) {
 858                 printf("%d: Error with start call\n", __LINE__);
 859                 return -1;
 860         }
 861
 862         const uint32_t XSTATS_MAX = 1024;
 863
 864         uint32_t i;
 865         uint32_t ids[XSTATS_MAX];
 866         uint64_t values[XSTATS_MAX];
 867         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
 868
 869         for (i = 0; i < XSTATS_MAX; i++)
 870                 ids[i] = i;
 871
 872         /* Device names / values */
 873         int ret = rte_event_dev_xstats_names_get(evdev,
 874                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 875                                         0, xstats_names, ids, XSTATS_MAX);
 876         if (ret != 6) {
 877                 printf("%d: expected 6 stats, got return %d\n", __LINE__, ret);
 878                 return -1;
 879         }
 880         ret = rte_event_dev_xstats_get(evdev,
 881                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 882                                         0, ids, values, ret);
 883         if (ret != 6) {
 884                 printf("%d: expected 6 stats, got return %d\n", __LINE__, ret);
 885                 return -1;
 886         }
 887
 888         /* Port names / values */
 889         ret = rte_event_dev_xstats_names_get(evdev,
 890                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
 891                                         xstats_names, ids, XSTATS_MAX);
 892         if (ret != 21) {
 893                 printf("%d: expected 21 stats, got return %d\n", __LINE__, ret);
 894                 return -1;
 895         }
 896         ret = rte_event_dev_xstats_get(evdev,
 897                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
 898                                         ids, values, ret);
 899         if (ret != 21) {
 900                 printf("%d: expected 21 stats, got return %d\n", __LINE__, ret);
 901                 return -1;
 902         }
 903
 904         /* Queue names / values */
 905         ret = rte_event_dev_xstats_names_get(evdev,
 906                                         RTE_EVENT_DEV_XSTATS_QUEUE,
 907                                         0, xstats_names, ids, XSTATS_MAX);
 908         if (ret != 16) {
 909                 printf("%d: expected 16 stats, got return %d\n", __LINE__, ret);
 910                 return -1;
 911         }
 912
 913         /* NEGATIVE TEST: with wrong queue passed, 0 stats should be returned */
 914         ret = rte_event_dev_xstats_get(evdev,
 915                                         RTE_EVENT_DEV_XSTATS_QUEUE,
 916                                         1, ids, values, ret);
 917         if (ret != -EINVAL) {
 918                 printf("%d: expected 0 stats, got return %d\n", __LINE__, ret);
 919                 return -1;
 920         }
 921
 922         ret = rte_event_dev_xstats_get(evdev,
 923                                         RTE_EVENT_DEV_XSTATS_QUEUE,
 924                                         0, ids, values, ret);
 925         if (ret != 16) {
 926                 printf("%d: expected 16 stats, got return %d\n", __LINE__, ret);
 927                 return -1;
 928         }
 929
 930         /* enqueue packets to check values */
 931         for (i = 0; i < 3; i++) {
 932                 struct rte_event ev;
 933                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
 934                 if (!arp) {
 935                         printf("%d: gen of pkt failed\n", __LINE__);
 936                         return -1;
 937                 }
 938                 ev.queue_id = t->qid[i];
 939                 ev.op = RTE_EVENT_OP_NEW;
 940                 ev.mbuf = arp;
 941                 ev.flow_id = 7;
 942                 arp->seqn = i;
 943
 944                 int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
 945                 if (err != 1) {
 946                         printf("%d: Failed to enqueue\n", __LINE__);
 947                         return -1;
 948                 }
 949         }
 950
 951         rte_service_run_iter_on_app_lcore(t->service_id, 1);
 952
 953         /* Device names / values */
 954         int num_stats = rte_event_dev_xstats_names_get(evdev,
 955                                         RTE_EVENT_DEV_XSTATS_DEVICE, 0,
 956                                         xstats_names, ids, XSTATS_MAX);
 957         if (num_stats < 0)
 958                 goto fail;
 959         ret = rte_event_dev_xstats_get(evdev,
 960                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 961                                         0, ids, values, num_stats);
 962         static const uint64_t expected[] = {3, 3, 0, 1, 0, 0};
 963         for (i = 0; (signed int)i < ret; i++) {
 964                 if (expected[i] != values[i]) {
 965                         printf(
 966                                 "%d Error xstat %d (id %d) %s : %"PRIu64
 967                                 ", expect %"PRIu64"\n",
 968                                 __LINE__, i, ids[i], xstats_names[i].name,
 969                                 values[i], expected[i]);
 970                         goto fail;
 971                 }
 972         }
 973
 974         ret = rte_event_dev_xstats_reset(evdev, RTE_EVENT_DEV_XSTATS_DEVICE,
 975                                         0, NULL, 0);
 976
 977         /* ensure reset statistics are zero-ed */
 978         static const uint64_t expected_zero[] = {0, 0, 0, 0, 0, 0};
 979         ret = rte_event_dev_xstats_get(evdev,
 980                                         RTE_EVENT_DEV_XSTATS_DEVICE,
 981                                         0, ids, values, num_stats);
 982         for (i = 0; (signed int)i < ret; i++) {
 983                 if (expected_zero[i] != values[i]) {
 984                         printf(
 985                                 "%d Error, xstat %d (id %d) %s : %"PRIu64
 986                                 ", expect %"PRIu64"\n",
 987                                 __LINE__, i, ids[i], xstats_names[i].name,
 988                                 values[i], expected_zero[i]);
 989                         goto fail;
 990                 }
 991         }
 992
 993         /* port reset checks */
 994         num_stats = rte_event_dev_xstats_names_get(evdev,
 995                                         RTE_EVENT_DEV_XSTATS_PORT, 0,
 996                                         xstats_names, ids, XSTATS_MAX);
 997         if (num_stats < 0)
 998                 goto fail;
 999         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_PORT,
1000                                         0, ids, values, num_stats);
1001
1002         static const uint64_t port_expected[] = {
1003                 3 /* rx */,
1004                 0 /* tx */,
1005                 0 /* drop */,
1006                 0 /* inflights */,
1007                 0 /* avg pkt cycles */,
1008                 29 /* credits */,
1009                 0 /* rx ring used */,
1010                 4096 /* rx ring free */,
1011                 0 /* cq ring used */,
1012                 32 /* cq ring free */,
1013                 0 /* dequeue calls */,
1014                 /* 10 dequeue burst buckets */
1015                 0, 0, 0, 0, 0,
1016                 0, 0, 0, 0, 0,
1017         };
1018         if (ret != RTE_DIM(port_expected)) {
1019                 printf(
1020                         "%s %d: wrong number of port stats (%d), expected %zu\n",
1021                         __func__, __LINE__, ret, RTE_DIM(port_expected));
1022         }
1023
1024         for (i = 0; (signed int)i < ret; i++) {
1025                 if (port_expected[i] != values[i]) {
1026                         printf(
1027                                 "%s : %d: Error stat %s is %"PRIu64
1028                                 ", expected %"PRIu64"\n",
1029                                 __func__, __LINE__, xstats_names[i].name,
1030                                 values[i], port_expected[i]);
1031                         goto fail;
1032                 }
1033         }
1034
1035         ret = rte_event_dev_xstats_reset(evdev, RTE_EVENT_DEV_XSTATS_PORT,
1036                                         0, NULL, 0);
1037
1038         /* ensure reset statistics are zero-ed */
1039         static const uint64_t port_expected_zero[] = {
1040                 0 /* rx */,
1041                 0 /* tx */,
1042                 0 /* drop */,
1043                 0 /* inflights */,
1044                 0 /* avg pkt cycles */,
1045                 29 /* credits */,
1046                 0 /* rx ring used */,
1047                 4096 /* rx ring free */,
1048                 0 /* cq ring used */,
1049                 32 /* cq ring free */,
1050                 0 /* dequeue calls */,
1051                 /* 10 dequeue burst buckets */
1052                 0, 0, 0, 0, 0,
1053                 0, 0, 0, 0, 0,
1054         };
1055         ret = rte_event_dev_xstats_get(evdev,
1056                                         RTE_EVENT_DEV_XSTATS_PORT,
1057                                         0, ids, values, num_stats);
1058         for (i = 0; (signed int)i < ret; i++) {
1059                 if (port_expected_zero[i] != values[i]) {
1060                         printf(
1061                                 "%d, Error, xstat %d (id %d) %s : %"PRIu64
1062                                 ", expect %"PRIu64"\n",
1063                                 __LINE__, i, ids[i], xstats_names[i].name,
1064                                 values[i], port_expected_zero[i]);
1065                         goto fail;
1066                 }
1067         }
1068
1069         /* QUEUE STATS TESTS */
1070         num_stats = rte_event_dev_xstats_names_get(evdev,
1071                                                 RTE_EVENT_DEV_XSTATS_QUEUE, 0,
1072                                                 xstats_names, ids, XSTATS_MAX);
1073         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE,
1074                                         0, ids, values, num_stats);
1075         if (ret < 0) {
1076                 printf("xstats get returned %d\n", ret);
1077                 goto fail;
1078         }
1079         if ((unsigned int)ret > XSTATS_MAX)
1080                 printf("%s %d: more xstats available than space\n",
1081                                 __func__, __LINE__);
1082
1083         static const uint64_t queue_expected[] = {
1084                 3 /* rx */,
1085                 3 /* tx */,
1086                 0 /* drop */,
1087                 3 /* inflights */,
1088                 0, 0, 0, 0, /* iq 0, 1, 2, 3 used */
1089                 /* QID-to-Port: pinned_flows, packets */
1090                 0, 0,
1091                 0, 0,
1092                 1, 3,
1093                 0, 0,
1094         };
1095         for (i = 0; (signed int)i < ret; i++) {
1096                 if (queue_expected[i] != values[i]) {
1097                         printf(
1098                                 "%d, Error, xstat %d (id %d) %s : %"PRIu64
1099                                 ", expect %"PRIu64"\n",
1100                                 __LINE__, i, ids[i], xstats_names[i].name,
1101                                 values[i], queue_expected[i]);
1102                         goto fail;
1103                 }
1104         }
1105
1106         /* Reset the queue stats here */
1107         ret = rte_event_dev_xstats_reset(evdev,
1108                                         RTE_EVENT_DEV_XSTATS_QUEUE, 0,
1109                                         NULL,
1110                                         0);
1111
1112         /* Verify that the resetable stats are reset, and others are not */
1113         static const uint64_t queue_expected_zero[] = {
1114                 0 /* rx */,
1115                 0 /* tx */,
1116                 0 /* drop */,
1117                 3 /* inflight */,
1118                 0, 0, 0, 0, /* 4 iq used */
1119                 /* QID-to-Port: pinned_flows, packets */
1120                 0, 0,
1121                 0, 0,
1122                 1, 0,
1123                 0, 0,
1124         };
1125
1126         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE, 0,
1127                                         ids, values, num_stats);
1128         int fails = 0;
1129         for (i = 0; (signed int)i < ret; i++) {
1130                 if (queue_expected_zero[i] != values[i]) {
1131                         printf(
1132                                 "%d, Error, xstat %d (id %d) %s : %"PRIu64
1133                                 ", expect %"PRIu64"\n",
1134                                 __LINE__, i, ids[i], xstats_names[i].name,
1135                                 values[i], queue_expected_zero[i]);
1136                         fails++;
1137                 }
1138         }
1139         if (fails) {
1140                 printf("%d : %d of values were not as expected above\n",
1141                                 __LINE__, fails);
1142                 goto fail;
1143         }
1144
1145         cleanup(t);
1146         return 0;
1147
1148 fail:
1149         rte_event_dev_dump(0, stdout);
1150         cleanup(t);
1151         return -1;
1152 }
1153
1154
1155 static int
1156 xstats_id_abuse_tests(struct test *t)
1157 {
1158         int err;
1159         const uint32_t XSTATS_MAX = 1024;
1160         const uint32_t link_port = 2;
1161
1162         uint32_t ids[XSTATS_MAX];
1163         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
1164
1165         /* Create instance with 4 ports */
1166         if (init(t, 1, 4) < 0 ||
1167                         create_ports(t, 4) < 0 ||
1168                         create_atomic_qids(t, 1) < 0) {
1169                 printf("%d: Error initializing device\n", __LINE__);
1170                 goto fail;
1171         }
1172
1173         err = rte_event_port_link(evdev, t->port[link_port], NULL, NULL, 0);
1174         if (err != 1) {
1175                 printf("%d: error mapping lb qid\n", __LINE__);
1176                 goto fail;
1177         }
1178
1179         if (rte_event_dev_start(evdev) < 0) {
1180                 printf("%d: Error with start call\n", __LINE__);
1181                 goto fail;
1182         }
1183
1184         /* no test for device, as it ignores the port/q number */
1185         int num_stats = rte_event_dev_xstats_names_get(evdev,
1186                                         RTE_EVENT_DEV_XSTATS_PORT,
1187                                         UINT8_MAX-1, xstats_names, ids,
1188                                         XSTATS_MAX);
1189         if (num_stats != 0) {
1190                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1191                                 0, num_stats);
1192                 goto fail;
1193         }
1194
1195         num_stats = rte_event_dev_xstats_names_get(evdev,
1196                                         RTE_EVENT_DEV_XSTATS_QUEUE,
1197                                         UINT8_MAX-1, xstats_names, ids,
1198                                         XSTATS_MAX);
1199         if (num_stats != 0) {
1200                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1201                                 0, num_stats);
1202                 goto fail;
1203         }
1204
1205         cleanup(t);
1206         return 0;
1207 fail:
1208         cleanup(t);
1209         return -1;
1210 }
1211
1212 static int
1213 port_reconfig_credits(struct test *t)
1214 {
1215         if (init(t, 1, 1) < 0) {
1216                 printf("%d: Error initializing device\n", __LINE__);
1217                 return -1;
1218         }
1219
1220         uint32_t i;
1221         const uint32_t NUM_ITERS = 32;
1222         for (i = 0; i < NUM_ITERS; i++) {
1223                 const struct rte_event_queue_conf conf = {
1224                         .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1225                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1226                         .nb_atomic_flows = 1024,
1227                         .nb_atomic_order_sequences = 1024,
1228                 };
1229                 if (rte_event_queue_setup(evdev, 0, &conf) < 0) {
1230                         printf("%d: error creating qid\n", __LINE__);
1231                         return -1;
1232                 }
1233                 t->qid[0] = 0;
1234
1235                 static const struct rte_event_port_conf port_conf = {
1236                                 .new_event_threshold = 128,
1237                                 .dequeue_depth = 32,
1238                                 .enqueue_depth = 64,
1239                 };
1240                 if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
1241                         printf("%d Error setting up port\n", __LINE__);
1242                         return -1;
1243                 }
1244
1245                 int links = rte_event_port_link(evdev, 0, NULL, NULL, 0);
1246                 if (links != 1) {
1247                         printf("%d: error mapping lb qid\n", __LINE__);
1248                         goto fail;
1249                 }
1250
1251                 if (rte_event_dev_start(evdev) < 0) {
1252                         printf("%d: Error with start call\n", __LINE__);
1253                         goto fail;
1254                 }
1255
1256                 const uint32_t NPKTS = 1;
1257                 uint32_t j;
1258                 for (j = 0; j < NPKTS; j++) {
1259                         struct rte_event ev;
1260                         struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1261                         if (!arp) {
1262                                 printf("%d: gen of pkt failed\n", __LINE__);
1263                                 goto fail;
1264                         }
1265                         ev.queue_id = t->qid[0];
1266                         ev.op = RTE_EVENT_OP_NEW;
1267                         ev.mbuf = arp;
1268                         int err = rte_event_enqueue_burst(evdev, 0, &ev, 1);
1269                         if (err != 1) {
1270                                 printf("%d: Failed to enqueue\n", __LINE__);
1271                                 rte_event_dev_dump(0, stdout);
1272                                 goto fail;
1273                         }
1274                 }
1275
1276                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
1277
1278                 struct rte_event ev[NPKTS];
1279                 int deq = rte_event_dequeue_burst(evdev, t->port[0], ev,
1280                                                         NPKTS, 0);
1281                 if (deq != 1)
1282                         printf("%d error; no packet dequeued\n", __LINE__);
1283
1284                 /* let cleanup below stop the device on last iter */
1285                 if (i != NUM_ITERS-1)
1286                         rte_event_dev_stop(evdev);
1287         }
1288
1289         cleanup(t);
1290         return 0;
1291 fail:
1292         cleanup(t);
1293         return -1;
1294 }
1295
1296 static int
1297 port_single_lb_reconfig(struct test *t)
1298 {
1299         if (init(t, 2, 2) < 0) {
1300                 printf("%d: Error initializing device\n", __LINE__);
1301                 goto fail;
1302         }
1303
1304         static const struct rte_event_queue_conf conf_lb_atomic = {
1305                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1306                 .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1307                 .nb_atomic_flows = 1024,
1308                 .nb_atomic_order_sequences = 1024,
1309         };
1310         if (rte_event_queue_setup(evdev, 0, &conf_lb_atomic) < 0) {
1311                 printf("%d: error creating qid\n", __LINE__);
1312                 goto fail;
1313         }
1314
1315         static const struct rte_event_queue_conf conf_single_link = {
1316                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1317                 .event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK,
1318         };
1319         if (rte_event_queue_setup(evdev, 1, &conf_single_link) < 0) {
1320                 printf("%d: error creating qid\n", __LINE__);
1321                 goto fail;
1322         }
1323
1324         struct rte_event_port_conf port_conf = {
1325                 .new_event_threshold = 128,
1326                 .dequeue_depth = 32,
1327                 .enqueue_depth = 64,
1328         };
1329         if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
1330                 printf("%d Error setting up port\n", __LINE__);
1331                 goto fail;
1332         }
1333         if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
1334                 printf("%d Error setting up port\n", __LINE__);
1335                 goto fail;
1336         }
1337
1338         /* link port to lb queue */
1339         uint8_t queue_id = 0;
1340         if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
1341                 printf("%d: error creating link for qid\n", __LINE__);
1342                 goto fail;
1343         }
1344
1345         int ret = rte_event_port_unlink(evdev, 0, &queue_id, 1);
1346         if (ret != 1) {
1347                 printf("%d: Error unlinking lb port\n", __LINE__);
1348                 goto fail;
1349         }
1350
1351         queue_id = 1;
1352         if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
1353                 printf("%d: error creating link for qid\n", __LINE__);
1354                 goto fail;
1355         }
1356
1357         queue_id = 0;
1358         int err = rte_event_port_link(evdev, 1, &queue_id, NULL, 1);
1359         if (err != 1) {
1360                 printf("%d: error mapping lb qid\n", __LINE__);
1361                 goto fail;
1362         }
1363
1364         if (rte_event_dev_start(evdev) < 0) {
1365                 printf("%d: Error with start call\n", __LINE__);
1366                 goto fail;
1367         }
1368
1369         cleanup(t);
1370         return 0;
1371 fail:
1372         cleanup(t);
1373         return -1;
1374 }
1375
1376 static int
1377 xstats_brute_force(struct test *t)
1378 {
1379         uint32_t i;
1380         const uint32_t XSTATS_MAX = 1024;
1381         uint32_t ids[XSTATS_MAX];
1382         uint64_t values[XSTATS_MAX];
1383         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
1384
1385
1386         /* Create instance with 4 ports */
1387         if (init(t, 1, 4) < 0 ||
1388                         create_ports(t, 4) < 0 ||
1389                         create_atomic_qids(t, 1) < 0) {
1390                 printf("%d: Error initializing device\n", __LINE__);
1391                 return -1;
1392         }
1393
1394         int err = rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1395         if (err != 1) {
1396                 printf("%d: error mapping lb qid\n", __LINE__);
1397                 goto fail;
1398         }
1399
1400         if (rte_event_dev_start(evdev) < 0) {
1401                 printf("%d: Error with start call\n", __LINE__);
1402                 goto fail;
1403         }
1404
1405         for (i = 0; i < XSTATS_MAX; i++)
1406                 ids[i] = i;
1407
1408         for (i = 0; i < 3; i++) {
1409                 uint32_t mode = RTE_EVENT_DEV_XSTATS_DEVICE + i;
1410                 uint32_t j;
1411                 for (j = 0; j < UINT8_MAX; j++) {
1412                         rte_event_dev_xstats_names_get(evdev, mode,
1413                                 j, xstats_names, ids, XSTATS_MAX);
1414
1415                         rte_event_dev_xstats_get(evdev, mode, j, ids,
1416                                                  values, XSTATS_MAX);
1417                 }
1418         }
1419
1420         cleanup(t);
1421         return 0;
1422 fail:
1423         cleanup(t);
1424         return -1;
1425 }
1426
1427 static int
1428 xstats_id_reset_tests(struct test *t)
1429 {
1430         const int wrk_enq = 2;
1431         int err;
1432
1433         /* Create instance with 4 ports */
1434         if (init(t, 1, 4) < 0 ||
1435                         create_ports(t, 4) < 0 ||
1436                         create_atomic_qids(t, 1) < 0) {
1437                 printf("%d: Error initializing device\n", __LINE__);
1438                 return -1;
1439         }
1440
1441         /* CQ mapping to QID */
1442         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
1443         if (err != 1) {
1444                 printf("%d: error mapping lb qid\n", __LINE__);
1445                 goto fail;
1446         }
1447
1448         if (rte_event_dev_start(evdev) < 0) {
1449                 printf("%d: Error with start call\n", __LINE__);
1450                 goto fail;
1451         }
1452
1453 #define XSTATS_MAX 1024
1454         int ret;
1455         uint32_t i;
1456         uint32_t ids[XSTATS_MAX];
1457         uint64_t values[XSTATS_MAX];
1458         struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX];
1459
1460         for (i = 0; i < XSTATS_MAX; i++)
1461                 ids[i] = i;
1462
1463 #define NUM_DEV_STATS 6
1464         /* Device names / values */
1465         int num_stats = rte_event_dev_xstats_names_get(evdev,
1466                                         RTE_EVENT_DEV_XSTATS_DEVICE,
1467                                         0, xstats_names, ids, XSTATS_MAX);
1468         if (num_stats != NUM_DEV_STATS) {
1469                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1470                                 NUM_DEV_STATS, num_stats);
1471                 goto fail;
1472         }
1473         ret = rte_event_dev_xstats_get(evdev,
1474                                         RTE_EVENT_DEV_XSTATS_DEVICE,
1475                                         0, ids, values, num_stats);
1476         if (ret != NUM_DEV_STATS) {
1477                 printf("%d: expected %d stats, got return %d\n", __LINE__,
1478                                 NUM_DEV_STATS, ret);
1479                 goto fail;
1480         }
1481
1482 #define NPKTS 7
1483         for (i = 0; i < NPKTS; i++) {
1484                 struct rte_event ev;
1485                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1486                 if (!arp) {
1487                         printf("%d: gen of pkt failed\n", __LINE__);
1488                         goto fail;
1489                 }
1490                 ev.queue_id = t->qid[i];
1491                 ev.op = RTE_EVENT_OP_NEW;
1492                 ev.mbuf = arp;
1493                 arp->seqn = i;
1494
1495                 int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
1496                 if (err != 1) {
1497                         printf("%d: Failed to enqueue\n", __LINE__);
1498                         goto fail;
1499                 }
1500         }
1501
1502         rte_service_run_iter_on_app_lcore(t->service_id, 1);
1503
1504         static const char * const dev_names[] = {
1505                 "dev_rx", "dev_tx", "dev_drop", "dev_sched_calls",
1506                 "dev_sched_no_iq_enq", "dev_sched_no_cq_enq",
1507         };
1508         uint64_t dev_expected[] = {NPKTS, NPKTS, 0, 1, 0, 0};
1509         for (i = 0; (int)i < ret; i++) {
1510                 unsigned int id;
1511                 uint64_t val = rte_event_dev_xstats_by_name_get(evdev,
1512                                                                 dev_names[i],
1513                                                                 &id);
1514                 if (id != i) {
1515                         printf("%d: %s id incorrect, expected %d got %d\n",
1516                                         __LINE__, dev_names[i], i, id);
1517                         goto fail;
1518                 }
1519                 if (val != dev_expected[i]) {
1520                         printf("%d: %s value incorrect, expected %"
1521                                 PRIu64" got %d\n", __LINE__, dev_names[i],
1522                                 dev_expected[i], id);
1523                         goto fail;
1524                 }
1525                 /* reset to zero */
1526                 int reset_ret = rte_event_dev_xstats_reset(evdev,
1527                                                 RTE_EVENT_DEV_XSTATS_DEVICE, 0,
1528                                                 &id,
1529                                                 1);
1530                 if (reset_ret) {
1531                         printf("%d: failed to reset successfully\n", __LINE__);
1532                         goto fail;
1533                 }
1534                 dev_expected[i] = 0;
1535                 /* check value again */
1536                 val = rte_event_dev_xstats_by_name_get(evdev, dev_names[i], 0);
1537                 if (val != dev_expected[i]) {
1538                         printf("%d: %s value incorrect, expected %"PRIu64
1539                                 " got %"PRIu64"\n", __LINE__, dev_names[i],
1540                                 dev_expected[i], val);
1541                         goto fail;
1542                 }
1543         };
1544
1545 /* 48 is stat offset from start of the devices whole xstats.
1546  * This WILL break every time we add a statistic to a port
1547  * or the device, but there is no other way to test
1548  */
1549 #define PORT_OFF 48
1550 /* num stats for the tested port. CQ size adds more stats to a port */
1551 #define NUM_PORT_STATS 21
1552 /* the port to test. */
1553 #define PORT 2
1554         num_stats = rte_event_dev_xstats_names_get(evdev,
1555                                         RTE_EVENT_DEV_XSTATS_PORT, PORT,
1556                                         xstats_names, ids, XSTATS_MAX);
1557         if (num_stats != NUM_PORT_STATS) {
1558                 printf("%d: expected %d stats, got return %d\n",
1559                         __LINE__, NUM_PORT_STATS, num_stats);
1560                 goto fail;
1561         }
1562         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_PORT, PORT,
1563                                         ids, values, num_stats);
1564
1565         if (ret != NUM_PORT_STATS) {
1566                 printf("%d: expected %d stats, got return %d\n",
1567                                 __LINE__, NUM_PORT_STATS, ret);
1568                 goto fail;
1569         }
1570         static const char * const port_names[] = {
1571                 "port_2_rx",
1572                 "port_2_tx",
1573                 "port_2_drop",
1574                 "port_2_inflight",
1575                 "port_2_avg_pkt_cycles",
1576                 "port_2_credits",
1577                 "port_2_rx_ring_used",
1578                 "port_2_rx_ring_free",
1579                 "port_2_cq_ring_used",
1580                 "port_2_cq_ring_free",
1581                 "port_2_dequeue_calls",
1582                 "port_2_dequeues_returning_0",
1583                 "port_2_dequeues_returning_1-4",
1584                 "port_2_dequeues_returning_5-8",
1585                 "port_2_dequeues_returning_9-12",
1586                 "port_2_dequeues_returning_13-16",
1587                 "port_2_dequeues_returning_17-20",
1588                 "port_2_dequeues_returning_21-24",
1589                 "port_2_dequeues_returning_25-28",
1590                 "port_2_dequeues_returning_29-32",
1591                 "port_2_dequeues_returning_33-36",
1592         };
1593         uint64_t port_expected[] = {
1594                 0, /* rx */
1595                 NPKTS, /* tx */
1596                 0, /* drop */
1597                 NPKTS, /* inflight */
1598                 0, /* avg pkt cycles */
1599                 0, /* credits */
1600                 0, /* rx ring used */
1601                 4096, /* rx ring free */
1602                 NPKTS,  /* cq ring used */
1603                 25, /* cq ring free */
1604                 0, /* dequeue zero calls */
1605                 0, 0, 0, 0, 0, /* 10 dequeue buckets */
1606                 0, 0, 0, 0, 0,
1607         };
1608         uint64_t port_expected_zero[] = {
1609                 0, /* rx */
1610                 0, /* tx */
1611                 0, /* drop */
1612                 NPKTS, /* inflight */
1613                 0, /* avg pkt cycles */
1614                 0, /* credits */
1615                 0, /* rx ring used */
1616                 4096, /* rx ring free */
1617                 NPKTS,  /* cq ring used */
1618                 25, /* cq ring free */
1619                 0, /* dequeue zero calls */
1620                 0, 0, 0, 0, 0, /* 10 dequeue buckets */
1621                 0, 0, 0, 0, 0,
1622         };
1623         if (RTE_DIM(port_expected) != NUM_PORT_STATS ||
1624                         RTE_DIM(port_names) != NUM_PORT_STATS) {
1625                 printf("%d: port array of wrong size\n", __LINE__);
1626                 goto fail;
1627         }
1628
1629         int failed = 0;
1630         for (i = 0; (int)i < ret; i++) {
1631                 unsigned int id;
1632                 uint64_t val = rte_event_dev_xstats_by_name_get(evdev,
1633                                                                 port_names[i],
1634                                                                 &id);
1635                 if (id != i + PORT_OFF) {
1636                         printf("%d: %s id incorrect, expected %d got %d\n",
1637                                         __LINE__, port_names[i], i+PORT_OFF,
1638                                         id);
1639                         failed = 1;
1640                 }
1641                 if (val != port_expected[i]) {
1642                         printf("%d: %s value incorrect, expected %"PRIu64
1643                                 " got %d\n", __LINE__, port_names[i],
1644                                 port_expected[i], id);
1645                         failed = 1;
1646                 }
1647                 /* reset to zero */
1648                 int reset_ret = rte_event_dev_xstats_reset(evdev,
1649                                                 RTE_EVENT_DEV_XSTATS_PORT, PORT,
1650                                                 &id,
1651                                                 1);
1652                 if (reset_ret) {
1653                         printf("%d: failed to reset successfully\n", __LINE__);
1654                         failed = 1;
1655                 }
1656                 /* check value again */
1657                 val = rte_event_dev_xstats_by_name_get(evdev, port_names[i], 0);
1658                 if (val != port_expected_zero[i]) {
1659                         printf("%d: %s value incorrect, expected %"PRIu64
1660                                 " got %"PRIu64"\n", __LINE__, port_names[i],
1661                                 port_expected_zero[i], val);
1662                         failed = 1;
1663                 }
1664         };
1665         if (failed)
1666                 goto fail;
1667
1668 /* num queue stats */
1669 #define NUM_Q_STATS 16
1670 /* queue offset from start of the devices whole xstats.
1671  * This will break every time we add a statistic to a device/port/queue
1672  */
1673 #define QUEUE_OFF 90
1674         const uint32_t queue = 0;
1675         num_stats = rte_event_dev_xstats_names_get(evdev,
1676                                         RTE_EVENT_DEV_XSTATS_QUEUE, queue,
1677                                         xstats_names, ids, XSTATS_MAX);
1678         if (num_stats != NUM_Q_STATS) {
1679                 printf("%d: expected %d stats, got return %d\n",
1680                         __LINE__, NUM_Q_STATS, num_stats);
1681                 goto fail;
1682         }
1683         ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE,
1684                                         queue, ids, values, num_stats);
1685         if (ret != NUM_Q_STATS) {
1686                 printf("%d: expected 21 stats, got return %d\n", __LINE__, ret);
1687                 goto fail;
1688         }
1689         static const char * const queue_names[] = {
1690                 "qid_0_rx",
1691                 "qid_0_tx",
1692                 "qid_0_drop",
1693                 "qid_0_inflight",
1694                 "qid_0_iq_0_used",
1695                 "qid_0_iq_1_used",
1696                 "qid_0_iq_2_used",
1697                 "qid_0_iq_3_used",
1698                 "qid_0_port_0_pinned_flows",
1699                 "qid_0_port_0_packets",
1700                 "qid_0_port_1_pinned_flows",
1701                 "qid_0_port_1_packets",
1702                 "qid_0_port_2_pinned_flows",
1703                 "qid_0_port_2_packets",
1704                 "qid_0_port_3_pinned_flows",
1705                 "qid_0_port_3_packets",
1706         };
1707         uint64_t queue_expected[] = {
1708                 7, /* rx */
1709                 7, /* tx */
1710                 0, /* drop */
1711                 7, /* inflight */
1712                 0, /* iq 0 used */
1713                 0, /* iq 1 used */
1714                 0, /* iq 2 used */
1715                 0, /* iq 3 used */
1716                 /* QID-to-Port: pinned_flows, packets */
1717                 0, 0,
1718                 0, 0,
1719                 1, 7,
1720                 0, 0,
1721         };
1722         uint64_t queue_expected_zero[] = {
1723                 0, /* rx */
1724                 0, /* tx */
1725                 0, /* drop */
1726                 7, /* inflight */
1727                 0, /* iq 0 used */
1728                 0, /* iq 1 used */
1729                 0, /* iq 2 used */
1730                 0, /* iq 3 used */
1731                 /* QID-to-Port: pinned_flows, packets */
1732                 0, 0,
1733                 0, 0,
1734                 1, 0,
1735                 0, 0,
1736         };
1737         if (RTE_DIM(queue_expected) != NUM_Q_STATS ||
1738                         RTE_DIM(queue_expected_zero) != NUM_Q_STATS ||
1739                         RTE_DIM(queue_names) != NUM_Q_STATS) {
1740                 printf("%d : queue array of wrong size\n", __LINE__);
1741                 goto fail;
1742         }
1743
1744         failed = 0;
1745         for (i = 0; (int)i < ret; i++) {
1746                 unsigned int id;
1747                 uint64_t val = rte_event_dev_xstats_by_name_get(evdev,
1748                                                                 queue_names[i],
1749                                                                 &id);
1750                 if (id != i + QUEUE_OFF) {
1751                         printf("%d: %s id incorrect, expected %d got %d\n",
1752                                         __LINE__, queue_names[i], i+QUEUE_OFF,
1753                                         id);
1754                         failed = 1;
1755                 }
1756                 if (val != queue_expected[i]) {
1757                         printf("%d: %d: %s value , expected %"PRIu64
1758                                 " got %"PRIu64"\n", i, __LINE__,
1759                                 queue_names[i], queue_expected[i], val);
1760                         failed = 1;
1761                 }
1762                 /* reset to zero */
1763                 int reset_ret = rte_event_dev_xstats_reset(evdev,
1764                                                 RTE_EVENT_DEV_XSTATS_QUEUE,
1765                                                 queue, &id, 1);
1766                 if (reset_ret) {
1767                         printf("%d: failed to reset successfully\n", __LINE__);
1768                         failed = 1;
1769                 }
1770                 /* check value again */
1771                 val = rte_event_dev_xstats_by_name_get(evdev, queue_names[i],
1772                                                         0);
1773                 if (val != queue_expected_zero[i]) {
1774                         printf("%d: %s value incorrect, expected %"PRIu64
1775                                 " got %"PRIu64"\n", __LINE__, queue_names[i],
1776                                 queue_expected_zero[i], val);
1777                         failed = 1;
1778                 }
1779         };
1780
1781         if (failed)
1782                 goto fail;
1783
1784         cleanup(t);
1785         return 0;
1786 fail:
1787         cleanup(t);
1788         return -1;
1789 }
1790
1791 static int
1792 ordered_reconfigure(struct test *t)
1793 {
1794         if (init(t, 1, 1) < 0 ||
1795                         create_ports(t, 1) < 0) {
1796                 printf("%d: Error initializing device\n", __LINE__);
1797                 return -1;
1798         }
1799
1800         const struct rte_event_queue_conf conf = {
1801                         .schedule_type = RTE_SCHED_TYPE_ORDERED,
1802                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
1803                         .nb_atomic_flows = 1024,
1804                         .nb_atomic_order_sequences = 1024,
1805         };
1806
1807         if (rte_event_queue_setup(evdev, 0, &conf) < 0) {
1808                 printf("%d: error creating qid\n", __LINE__);
1809                 goto failed;
1810         }
1811
1812         if (rte_event_queue_setup(evdev, 0, &conf) < 0) {
1813                 printf("%d: error creating qid, for 2nd time\n", __LINE__);
1814                 goto failed;
1815         }
1816
1817         rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1818         if (rte_event_dev_start(evdev) < 0) {
1819                 printf("%d: Error with start call\n", __LINE__);
1820                 return -1;
1821         }
1822
1823         cleanup(t);
1824         return 0;
1825 failed:
1826         cleanup(t);
1827         return -1;
1828 }
1829
1830 static int
1831 qid_priorities(struct test *t)
1832 {
1833         /* Test works by having a CQ with enough empty space for all packets,
1834          * and enqueueing 3 packets to 3 QIDs. They must return based on the
1835          * priority of the QID, not the ingress order, to pass the test
1836          */
1837         unsigned int i;
1838         /* Create instance with 1 ports, and 3 qids */
1839         if (init(t, 3, 1) < 0 ||
1840                         create_ports(t, 1) < 0) {
1841                 printf("%d: Error initializing device\n", __LINE__);
1842                 return -1;
1843         }
1844
1845         for (i = 0; i < 3; i++) {
1846                 /* Create QID */
1847                 const struct rte_event_queue_conf conf = {
1848                         .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1849                         /* increase priority (0 == highest), as we go */
1850                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL - i,
1851                         .nb_atomic_flows = 1024,
1852                         .nb_atomic_order_sequences = 1024,
1853                 };
1854
1855                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
1856                         printf("%d: error creating qid %d\n", __LINE__, i);
1857                         return -1;
1858                 }
1859                 t->qid[i] = i;
1860         }
1861         t->nb_qids = i;
1862         /* map all QIDs to port */
1863         rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1864
1865         if (rte_event_dev_start(evdev) < 0) {
1866                 printf("%d: Error with start call\n", __LINE__);
1867                 return -1;
1868         }
1869
1870         /* enqueue 3 packets, setting seqn and QID to check priority */
1871         for (i = 0; i < 3; i++) {
1872                 struct rte_event ev;
1873                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
1874                 if (!arp) {
1875                         printf("%d: gen of pkt failed\n", __LINE__);
1876                         return -1;
1877                 }
1878                 ev.queue_id = t->qid[i];
1879                 ev.op = RTE_EVENT_OP_NEW;
1880                 ev.mbuf = arp;
1881                 arp->seqn = i;
1882
1883                 int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1);
1884                 if (err != 1) {
1885                         printf("%d: Failed to enqueue\n", __LINE__);
1886                         return -1;
1887                 }
1888         }
1889
1890         rte_service_run_iter_on_app_lcore(t->service_id, 1);
1891
1892         /* dequeue packets, verify priority was upheld */
1893         struct rte_event ev[32];
1894         uint32_t deq_pkts =
1895                 rte_event_dequeue_burst(evdev, t->port[0], ev, 32, 0);
1896         if (deq_pkts != 3) {
1897                 printf("%d: failed to deq packets\n", __LINE__);
1898                 rte_event_dev_dump(evdev, stdout);
1899                 return -1;
1900         }
1901         for (i = 0; i < 3; i++) {
1902                 if (ev[i].mbuf->seqn != 2-i) {
1903                         printf(
1904                                 "%d: qid priority test: seqn %d incorrectly prioritized\n",
1905                                         __LINE__, i);
1906                 }
1907         }
1908
1909         cleanup(t);
1910         return 0;
1911 }
1912
1913 static int
1914 unlink_in_progress(struct test *t)
1915 {
1916         /* Test unlinking API, in particular that when an unlink request has
1917          * not yet been seen by the scheduler thread, that the
1918          * unlink_in_progress() function returns the number of unlinks.
1919          */
1920         unsigned int i;
1921         /* Create instance with 1 ports, and 3 qids */
1922         if (init(t, 3, 1) < 0 ||
1923                         create_ports(t, 1) < 0) {
1924                 printf("%d: Error initializing device\n", __LINE__);
1925                 return -1;
1926         }
1927
1928         for (i = 0; i < 3; i++) {
1929                 /* Create QID */
1930                 const struct rte_event_queue_conf conf = {
1931                         .schedule_type = RTE_SCHED_TYPE_ATOMIC,
1932                         /* increase priority (0 == highest), as we go */
1933                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL - i,
1934                         .nb_atomic_flows = 1024,
1935                         .nb_atomic_order_sequences = 1024,
1936                 };
1937
1938                 if (rte_event_queue_setup(evdev, i, &conf) < 0) {
1939                         printf("%d: error creating qid %d\n", __LINE__, i);
1940                         return -1;
1941                 }
1942                 t->qid[i] = i;
1943         }
1944         t->nb_qids = i;
1945         /* map all QIDs to port */
1946         rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
1947
1948         if (rte_event_dev_start(evdev) < 0) {
1949                 printf("%d: Error with start call\n", __LINE__);
1950                 return -1;
1951         }
1952
1953         /* unlink all ports to have outstanding unlink requests */
1954         int ret = rte_event_port_unlink(evdev, t->port[0], NULL, 0);
1955         if (ret < 0) {
1956                 printf("%d: Failed to unlink queues\n", __LINE__);
1957                 return -1;
1958         }
1959
1960         /* get active unlinks here, expect 3 */
1961         int unlinks_in_progress =
1962                 rte_event_port_unlinks_in_progress(evdev, t->port[0]);
1963         if (unlinks_in_progress != 3) {
1964                 printf("%d: Expected num unlinks in progress == 3, got %d\n",
1965                                 __LINE__, unlinks_in_progress);
1966                 return -1;
1967         }
1968
1969         /* run scheduler service on this thread to ack the unlinks */
1970         rte_service_run_iter_on_app_lcore(t->service_id, 1);
1971
1972         /* active unlinks expected as 0 as scheduler thread has acked */
1973         unlinks_in_progress =
1974                 rte_event_port_unlinks_in_progress(evdev, t->port[0]);
1975         if (unlinks_in_progress != 0) {
1976                 printf("%d: Expected num unlinks in progress == 0, got %d\n",
1977                                 __LINE__, unlinks_in_progress);
1978         }
1979
1980         cleanup(t);
1981         return 0;
1982 }
1983
1984 static int
1985 load_balancing(struct test *t)
1986 {
1987         const int rx_enq = 0;
1988         int err;
1989         uint32_t i;
1990
1991         if (init(t, 1, 4) < 0 ||
1992                         create_ports(t, 4) < 0 ||
1993                         create_atomic_qids(t, 1) < 0) {
1994                 printf("%d: Error initializing device\n", __LINE__);
1995                 return -1;
1996         }
1997
1998         for (i = 0; i < 3; i++) {
1999                 /* map port 1 - 3 inclusive */
2000                 if (rte_event_port_link(evdev, t->port[i+1], &t->qid[0],
2001                                 NULL, 1) != 1) {
2002                         printf("%d: error mapping qid to port %d\n",
2003                                         __LINE__, i);
2004                         return -1;
2005                 }
2006         }
2007
2008         if (rte_event_dev_start(evdev) < 0) {
2009                 printf("%d: Error with start call\n", __LINE__);
2010                 return -1;
2011         }
2012
2013         /************** FORWARD ****************/
2014         /*
2015          * Create a set of flows that test the load-balancing operation of the
2016          * implementation. Fill CQ 0 and 1 with flows 0 and 1, and test
2017          * with a new flow, which should be sent to the 3rd mapped CQ
2018          */
2019         static uint32_t flows[] = {0, 1, 1, 0, 0, 2, 2, 0, 2};
2020
2021         for (i = 0; i < RTE_DIM(flows); i++) {
2022                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2023                 if (!arp) {
2024                         printf("%d: gen of pkt failed\n", __LINE__);
2025                         return -1;
2026                 }
2027
2028                 struct rte_event ev = {
2029                                 .op = RTE_EVENT_OP_NEW,
2030                                 .queue_id = t->qid[0],
2031                                 .flow_id = flows[i],
2032                                 .mbuf = arp,
2033                 };
2034                 /* generate pkt and enqueue */
2035                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2036                 if (err != 1) {
2037                         printf("%d: Failed to enqueue\n", __LINE__);
2038                         return -1;
2039                 }
2040         }
2041
2042         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2043
2044         struct test_event_dev_stats stats;
2045         err = test_event_dev_stats_get(evdev, &stats);
2046         if (err) {
2047                 printf("%d: failed to get stats\n", __LINE__);
2048                 return -1;
2049         }
2050
2051         if (stats.port_inflight[1] != 4) {
2052                 printf("%d:%s: port 1 inflight not correct\n", __LINE__,
2053                                 __func__);
2054                 return -1;
2055         }
2056         if (stats.port_inflight[2] != 2) {
2057                 printf("%d:%s: port 2 inflight not correct\n", __LINE__,
2058                                 __func__);
2059                 return -1;
2060         }
2061         if (stats.port_inflight[3] != 3) {
2062                 printf("%d:%s: port 3 inflight not correct\n", __LINE__,
2063                                 __func__);
2064                 return -1;
2065         }
2066
2067         cleanup(t);
2068         return 0;
2069 }
2070
2071 static int
2072 load_balancing_history(struct test *t)
2073 {
2074         struct test_event_dev_stats stats = {0};
2075         const int rx_enq = 0;
2076         int err;
2077         uint32_t i;
2078
2079         /* Create instance with 1 atomic QID going to 3 ports + 1 prod port */
2080         if (init(t, 1, 4) < 0 ||
2081                         create_ports(t, 4) < 0 ||
2082                         create_atomic_qids(t, 1) < 0)
2083                 return -1;
2084
2085         /* CQ mapping to QID */
2086         if (rte_event_port_link(evdev, t->port[1], &t->qid[0], NULL, 1) != 1) {
2087                 printf("%d: error mapping port 1 qid\n", __LINE__);
2088                 return -1;
2089         }
2090         if (rte_event_port_link(evdev, t->port[2], &t->qid[0], NULL, 1) != 1) {
2091                 printf("%d: error mapping port 2 qid\n", __LINE__);
2092                 return -1;
2093         }
2094         if (rte_event_port_link(evdev, t->port[3], &t->qid[0], NULL, 1) != 1) {
2095                 printf("%d: error mapping port 3 qid\n", __LINE__);
2096                 return -1;
2097         }
2098         if (rte_event_dev_start(evdev) < 0) {
2099                 printf("%d: Error with start call\n", __LINE__);
2100                 return -1;
2101         }
2102
2103         /*
2104          * Create a set of flows that test the load-balancing operation of the
2105          * implementation. Fill CQ 0, 1 and 2 with flows 0, 1 and 2, drop
2106          * the packet from CQ 0, send in a new set of flows. Ensure that:
2107          *  1. The new flow 3 gets into the empty CQ0
2108          *  2. packets for existing flow gets added into CQ1
2109          *  3. Next flow 0 pkt is now onto CQ2, since CQ0 and CQ1 now contain
2110          *     more outstanding pkts
2111          *
2112          *  This test makes sure that when a flow ends (i.e. all packets
2113          *  have been completed for that flow), that the flow can be moved
2114          *  to a different CQ when new packets come in for that flow.
2115          */
2116         static uint32_t flows1[] = {0, 1, 1, 2};
2117
2118         for (i = 0; i < RTE_DIM(flows1); i++) {
2119                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2120                 struct rte_event ev = {
2121                                 .flow_id = flows1[i],
2122                                 .op = RTE_EVENT_OP_NEW,
2123                                 .queue_id = t->qid[0],
2124                                 .event_type = RTE_EVENT_TYPE_CPU,
2125                                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
2126                                 .mbuf = arp
2127                 };
2128
2129                 if (!arp) {
2130                         printf("%d: gen of pkt failed\n", __LINE__);
2131                         return -1;
2132                 }
2133                 arp->hash.rss = flows1[i];
2134                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2135                 if (err != 1) {
2136                         printf("%d: Failed to enqueue\n", __LINE__);
2137                         return -1;
2138                 }
2139         }
2140
2141         /* call the scheduler */
2142         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2143
2144         /* Dequeue the flow 0 packet from port 1, so that we can then drop */
2145         struct rte_event ev;
2146         if (!rte_event_dequeue_burst(evdev, t->port[1], &ev, 1, 0)) {
2147                 printf("%d: failed to dequeue\n", __LINE__);
2148                 return -1;
2149         }
2150         if (ev.mbuf->hash.rss != flows1[0]) {
2151                 printf("%d: unexpected flow received\n", __LINE__);
2152                 return -1;
2153         }
2154
2155         /* drop the flow 0 packet from port 1 */
2156         rte_event_enqueue_burst(evdev, t->port[1], &release_ev, 1);
2157
2158         /* call the scheduler */
2159         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2160
2161         /*
2162          * Set up the next set of flows, first a new flow to fill up
2163          * CQ 0, so that the next flow 0 packet should go to CQ2
2164          */
2165         static uint32_t flows2[] = { 3, 3, 3, 1, 1, 0 };
2166
2167         for (i = 0; i < RTE_DIM(flows2); i++) {
2168                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2169                 struct rte_event ev = {
2170                                 .flow_id = flows2[i],
2171                                 .op = RTE_EVENT_OP_NEW,
2172                                 .queue_id = t->qid[0],
2173                                 .event_type = RTE_EVENT_TYPE_CPU,
2174                                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
2175                                 .mbuf = arp
2176                 };
2177
2178                 if (!arp) {
2179                         printf("%d: gen of pkt failed\n", __LINE__);
2180                         return -1;
2181                 }
2182                 arp->hash.rss = flows2[i];
2183
2184                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2185                 if (err != 1) {
2186                         printf("%d: Failed to enqueue\n", __LINE__);
2187                         return -1;
2188                 }
2189         }
2190
2191         /* schedule */
2192         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2193
2194         err = test_event_dev_stats_get(evdev, &stats);
2195         if (err) {
2196                 printf("%d:failed to get stats\n", __LINE__);
2197                 return -1;
2198         }
2199
2200         /*
2201          * Now check the resulting inflights on each port.
2202          */
2203         if (stats.port_inflight[1] != 3) {
2204                 printf("%d:%s: port 1 inflight not correct\n", __LINE__,
2205                                 __func__);
2206                 printf("Inflights, ports 1, 2, 3: %u, %u, %u\n",
2207                                 (unsigned int)stats.port_inflight[1],
2208                                 (unsigned int)stats.port_inflight[2],
2209                                 (unsigned int)stats.port_inflight[3]);
2210                 return -1;
2211         }
2212         if (stats.port_inflight[2] != 4) {
2213                 printf("%d:%s: port 2 inflight not correct\n", __LINE__,
2214                                 __func__);
2215                 printf("Inflights, ports 1, 2, 3: %u, %u, %u\n",
2216                                 (unsigned int)stats.port_inflight[1],
2217                                 (unsigned int)stats.port_inflight[2],
2218                                 (unsigned int)stats.port_inflight[3]);
2219                 return -1;
2220         }
2221         if (stats.port_inflight[3] != 2) {
2222                 printf("%d:%s: port 3 inflight not correct\n", __LINE__,
2223                                 __func__);
2224                 printf("Inflights, ports 1, 2, 3: %u, %u, %u\n",
2225                                 (unsigned int)stats.port_inflight[1],
2226                                 (unsigned int)stats.port_inflight[2],
2227                                 (unsigned int)stats.port_inflight[3]);
2228                 return -1;
2229         }
2230
2231         for (i = 1; i <= 3; i++) {
2232                 struct rte_event ev;
2233                 while (rte_event_dequeue_burst(evdev, i, &ev, 1, 0))
2234                         rte_event_enqueue_burst(evdev, i, &release_ev, 1);
2235         }
2236         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2237
2238         cleanup(t);
2239         return 0;
2240 }
2241
2242 static int
2243 invalid_qid(struct test *t)
2244 {
2245         struct test_event_dev_stats stats;
2246         const int rx_enq = 0;
2247         int err;
2248         uint32_t i;
2249
2250         if (init(t, 1, 4) < 0 ||
2251                         create_ports(t, 4) < 0 ||
2252                         create_atomic_qids(t, 1) < 0) {
2253                 printf("%d: Error initializing device\n", __LINE__);
2254                 return -1;
2255         }
2256
2257         /* CQ mapping to QID */
2258         for (i = 0; i < 4; i++) {
2259                 err = rte_event_port_link(evdev, t->port[i], &t->qid[0],
2260                                 NULL, 1);
2261                 if (err != 1) {
2262                         printf("%d: error mapping port 1 qid\n", __LINE__);
2263                         return -1;
2264                 }
2265         }
2266
2267         if (rte_event_dev_start(evdev) < 0) {
2268                 printf("%d: Error with start call\n", __LINE__);
2269                 return -1;
2270         }
2271
2272         /*
2273          * Send in a packet with an invalid qid to the scheduler.
2274          * We should see the packed enqueued OK, but the inflights for
2275          * that packet should not be incremented, and the rx_dropped
2276          * should be incremented.
2277          */
2278         static uint32_t flows1[] = {20};
2279
2280         for (i = 0; i < RTE_DIM(flows1); i++) {
2281                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2282                 if (!arp) {
2283                         printf("%d: gen of pkt failed\n", __LINE__);
2284                         return -1;
2285                 }
2286
2287                 struct rte_event ev = {
2288                                 .op = RTE_EVENT_OP_NEW,
2289                                 .queue_id = t->qid[0] + flows1[i],
2290                                 .flow_id = i,
2291                                 .mbuf = arp,
2292                 };
2293                 /* generate pkt and enqueue */
2294                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2295                 if (err != 1) {
2296                         printf("%d: Failed to enqueue\n", __LINE__);
2297                         return -1;
2298                 }
2299         }
2300
2301         /* call the scheduler */
2302         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2303
2304         err = test_event_dev_stats_get(evdev, &stats);
2305         if (err) {
2306                 printf("%d: failed to get stats\n", __LINE__);
2307                 return -1;
2308         }
2309
2310         /*
2311          * Now check the resulting inflights on the port, and the rx_dropped.
2312          */
2313         if (stats.port_inflight[0] != 0) {
2314                 printf("%d:%s: port 1 inflight count not correct\n", __LINE__,
2315                                 __func__);
2316                 rte_event_dev_dump(evdev, stdout);
2317                 return -1;
2318         }
2319         if (stats.port_rx_dropped[0] != 1) {
2320                 printf("%d:%s: port 1 drops\n", __LINE__, __func__);
2321                 rte_event_dev_dump(evdev, stdout);
2322                 return -1;
2323         }
2324         /* each packet drop should only be counted in one place - port or dev */
2325         if (stats.rx_dropped != 0) {
2326                 printf("%d:%s: port 1 dropped count not correct\n", __LINE__,
2327                                 __func__);
2328                 rte_event_dev_dump(evdev, stdout);
2329                 return -1;
2330         }
2331
2332         cleanup(t);
2333         return 0;
2334 }
2335
2336 static int
2337 single_packet(struct test *t)
2338 {
2339         const uint32_t MAGIC_SEQN = 7321;
2340         struct rte_event ev;
2341         struct test_event_dev_stats stats;
2342         const int rx_enq = 0;
2343         const int wrk_enq = 2;
2344         int err;
2345
2346         /* Create instance with 4 ports */
2347         if (init(t, 1, 4) < 0 ||
2348                         create_ports(t, 4) < 0 ||
2349                         create_atomic_qids(t, 1) < 0) {
2350                 printf("%d: Error initializing device\n", __LINE__);
2351                 return -1;
2352         }
2353
2354         /* CQ mapping to QID */
2355         err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0);
2356         if (err != 1) {
2357                 printf("%d: error mapping lb qid\n", __LINE__);
2358                 cleanup(t);
2359                 return -1;
2360         }
2361
2362         if (rte_event_dev_start(evdev) < 0) {
2363                 printf("%d: Error with start call\n", __LINE__);
2364                 return -1;
2365         }
2366
2367         /************** Gen pkt and enqueue ****************/
2368         struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2369         if (!arp) {
2370                 printf("%d: gen of pkt failed\n", __LINE__);
2371                 return -1;
2372         }
2373
2374         ev.op = RTE_EVENT_OP_NEW;
2375         ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
2376         ev.mbuf = arp;
2377         ev.queue_id = 0;
2378         ev.flow_id = 3;
2379         arp->seqn = MAGIC_SEQN;
2380
2381         err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2382         if (err != 1) {
2383                 printf("%d: Failed to enqueue\n", __LINE__);
2384                 return -1;
2385         }
2386
2387         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2388
2389         err = test_event_dev_stats_get(evdev, &stats);
2390         if (err) {
2391                 printf("%d: failed to get stats\n", __LINE__);
2392                 return -1;
2393         }
2394
2395         if (stats.rx_pkts != 1 ||
2396                         stats.tx_pkts != 1 ||
2397                         stats.port_inflight[wrk_enq] != 1) {
2398                 printf("%d: Sched core didn't handle pkt as expected\n",
2399                                 __LINE__);
2400                 rte_event_dev_dump(evdev, stdout);
2401                 return -1;
2402         }
2403
2404         uint32_t deq_pkts;
2405
2406         deq_pkts = rte_event_dequeue_burst(evdev, t->port[wrk_enq], &ev, 1, 0);
2407         if (deq_pkts < 1) {
2408                 printf("%d: Failed to deq\n", __LINE__);
2409                 return -1;
2410         }
2411
2412         err = test_event_dev_stats_get(evdev, &stats);
2413         if (err) {
2414                 printf("%d: failed to get stats\n", __LINE__);
2415                 return -1;
2416         }
2417
2418         err = test_event_dev_stats_get(evdev, &stats);
2419         if (ev.mbuf->seqn != MAGIC_SEQN) {
2420                 printf("%d: magic sequence number not dequeued\n", __LINE__);
2421                 return -1;
2422         }
2423
2424         rte_pktmbuf_free(ev.mbuf);
2425         err = rte_event_enqueue_burst(evdev, t->port[wrk_enq], &release_ev, 1);
2426         if (err != 1) {
2427                 printf("%d: Failed to enqueue\n", __LINE__);
2428                 return -1;
2429         }
2430         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2431
2432         err = test_event_dev_stats_get(evdev, &stats);
2433         if (stats.port_inflight[wrk_enq] != 0) {
2434                 printf("%d: port inflight not correct\n", __LINE__);
2435                 return -1;
2436         }
2437
2438         cleanup(t);
2439         return 0;
2440 }
2441
2442 static int
2443 inflight_counts(struct test *t)
2444 {
2445         struct rte_event ev;
2446         struct test_event_dev_stats stats;
2447         const int rx_enq = 0;
2448         const int p1 = 1;
2449         const int p2 = 2;
2450         int err;
2451         int i;
2452
2453         /* Create instance with 4 ports */
2454         if (init(t, 2, 3) < 0 ||
2455                         create_ports(t, 3) < 0 ||
2456                         create_atomic_qids(t, 2) < 0) {
2457                 printf("%d: Error initializing device\n", __LINE__);
2458                 return -1;
2459         }
2460
2461         /* CQ mapping to QID */
2462         err = rte_event_port_link(evdev, t->port[p1], &t->qid[0], NULL, 1);
2463         if (err != 1) {
2464                 printf("%d: error mapping lb qid\n", __LINE__);
2465                 cleanup(t);
2466                 return -1;
2467         }
2468         err = rte_event_port_link(evdev, t->port[p2], &t->qid[1], NULL, 1);
2469         if (err != 1) {
2470                 printf("%d: error mapping lb qid\n", __LINE__);
2471                 cleanup(t);
2472                 return -1;
2473         }
2474
2475         if (rte_event_dev_start(evdev) < 0) {
2476                 printf("%d: Error with start call\n", __LINE__);
2477                 return -1;
2478         }
2479
2480         /************** FORWARD ****************/
2481 #define QID1_NUM 5
2482         for (i = 0; i < QID1_NUM; i++) {
2483                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2484
2485                 if (!arp) {
2486                         printf("%d: gen of pkt failed\n", __LINE__);
2487                         goto err;
2488                 }
2489
2490                 ev.queue_id =  t->qid[0];
2491                 ev.op = RTE_EVENT_OP_NEW;
2492                 ev.mbuf = arp;
2493                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2494                 if (err != 1) {
2495                         printf("%d: Failed to enqueue\n", __LINE__);
2496                         goto err;
2497                 }
2498         }
2499 #define QID2_NUM 3
2500         for (i = 0; i < QID2_NUM; i++) {
2501                 struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool);
2502
2503                 if (!arp) {
2504                         printf("%d: gen of pkt failed\n", __LINE__);
2505                         goto err;
2506                 }
2507                 ev.queue_id =  t->qid[1];
2508                 ev.op = RTE_EVENT_OP_NEW;
2509                 ev.mbuf = arp;
2510                 err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1);
2511                 if (err != 1) {
2512                         printf("%d: Failed to enqueue\n", __LINE__);
2513                         goto err;
2514                 }
2515         }
2516
2517         /* schedule */
2518         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2519
2520         err = test_event_dev_stats_get(evdev, &stats);
2521         if (err) {
2522                 printf("%d: failed to get stats\n", __LINE__);
2523                 goto err;
2524         }
2525
2526         if (stats.rx_pkts != QID1_NUM + QID2_NUM ||
2527                         stats.tx_pkts != QID1_NUM + QID2_NUM) {
2528                 printf("%d: Sched core didn't handle pkt as expected\n",
2529                                 __LINE__);
2530                 goto err;
2531         }
2532
2533         if (stats.port_inflight[p1] != QID1_NUM) {
2534                 printf("%d: %s port 1 inflight not correct\n", __LINE__,
2535                                 __func__);
2536                 goto err;
2537         }
2538         if (stats.port_inflight[p2] != QID2_NUM) {
2539                 printf("%d: %s port 2 inflight not correct\n", __LINE__,
2540                                 __func__);
2541                 goto err;
2542         }
2543
2544         /************** DEQUEUE INFLIGHT COUNT CHECKS  ****************/
2545         /* port 1 */
2546         struct rte_event events[QID1_NUM + QID2_NUM];
2547         uint32_t deq_pkts = rte_event_dequeue_burst(evdev, t->port[p1], events,
2548                         RTE_DIM(events), 0);
2549
2550         if (deq_pkts != QID1_NUM) {
2551                 printf("%d: Port 1: DEQUEUE inflight failed\n", __LINE__);
2552                 goto err;
2553         }
2554         err = test_event_dev_stats_get(evdev, &stats);
2555         if (stats.port_inflight[p1] != QID1_NUM) {
2556                 printf("%d: port 1 inflight decrement after DEQ != 0\n",
2557                                 __LINE__);
2558                 goto err;
2559         }
2560         for (i = 0; i < QID1_NUM; i++) {
2561                 err = rte_event_enqueue_burst(evdev, t->port[p1], &release_ev,
2562                                 1);
2563                 if (err != 1) {
2564                         printf("%d: %s rte enqueue of inf release failed\n",
2565                                 __LINE__, __func__);
2566                         goto err;
2567                 }
2568         }
2569
2570         /*
2571          * As the scheduler core decrements inflights, it needs to run to
2572          * process packets to act on the drop messages
2573          */
2574         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2575
2576         err = test_event_dev_stats_get(evdev, &stats);
2577         if (stats.port_inflight[p1] != 0) {
2578                 printf("%d: port 1 inflight NON NULL after DROP\n", __LINE__);
2579                 goto err;
2580         }
2581
2582         /* port2 */
2583         deq_pkts = rte_event_dequeue_burst(evdev, t->port[p2], events,
2584                         RTE_DIM(events), 0);
2585         if (deq_pkts != QID2_NUM) {
2586                 printf("%d: Port 2: DEQUEUE inflight failed\n", __LINE__);
2587                 goto err;
2588         }
2589         err = test_event_dev_stats_get(evdev, &stats);
2590         if (stats.port_inflight[p2] != QID2_NUM) {
2591                 printf("%d: port 1 inflight decrement after DEQ != 0\n",
2592                                 __LINE__);
2593                 goto err;
2594         }
2595         for (i = 0; i < QID2_NUM; i++) {
2596                 err = rte_event_enqueue_burst(evdev, t->port[p2], &release_ev,
2597                                 1);
2598                 if (err != 1) {
2599                         printf("%d: %s rte enqueue of inf release failed\n",
2600                                 __LINE__, __func__);
2601                         goto err;
2602                 }
2603         }
2604
2605         /*
2606          * As the scheduler core decrements inflights, it needs to run to
2607          * process packets to act on the drop messages
2608          */
2609         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2610
2611         err = test_event_dev_stats_get(evdev, &stats);
2612         if (stats.port_inflight[p2] != 0) {
2613                 printf("%d: port 2 inflight NON NULL after DROP\n", __LINE__);
2614                 goto err;
2615         }
2616         cleanup(t);
2617         return 0;
2618
2619 err:
2620         rte_event_dev_dump(evdev, stdout);
2621         cleanup(t);
2622         return -1;
2623 }
2624
2625 static int
2626 parallel_basic(struct test *t, int check_order)
2627 {
2628         const uint8_t rx_port = 0;
2629         const uint8_t w1_port = 1;
2630         const uint8_t w3_port = 3;
2631         const uint8_t tx_port = 4;
2632         int err;
2633         int i;
2634         uint32_t deq_pkts, j;
2635         struct rte_mbuf *mbufs[3];
2636         struct rte_mbuf *mbufs_out[3] = { 0 };
2637         const uint32_t MAGIC_SEQN = 1234;
2638
2639         /* Create instance with 4 ports */
2640         if (init(t, 2, tx_port + 1) < 0 ||
2641                         create_ports(t, tx_port + 1) < 0 ||
2642                         (check_order ?  create_ordered_qids(t, 1) :
2643                                 create_unordered_qids(t, 1)) < 0 ||
2644                         create_directed_qids(t, 1, &tx_port)) {
2645                 printf("%d: Error initializing device\n", __LINE__);
2646                 return -1;
2647         }
2648
2649         /*
2650          * CQ mapping to QID
2651          * We need three ports, all mapped to the same ordered qid0. Then we'll
2652          * take a packet out to each port, re-enqueue in reverse order,
2653          * then make sure the reordering has taken place properly when we
2654          * dequeue from the tx_port.
2655          *
2656          * Simplified test setup diagram:
2657          *
2658          * rx_port        w1_port
2659          *        \     /         \
2660          *         qid0 - w2_port - qid1
2661          *              \         /     \
2662          *                w3_port        tx_port
2663          */
2664         /* CQ mapping to QID for LB ports (directed mapped on create) */
2665         for (i = w1_port; i <= w3_port; i++) {
2666                 err = rte_event_port_link(evdev, t->port[i], &t->qid[0], NULL,
2667                                 1);
2668                 if (err != 1) {
2669                         printf("%d: error mapping lb qid\n", __LINE__);
2670                         cleanup(t);
2671                         return -1;
2672                 }
2673         }
2674
2675         if (rte_event_dev_start(evdev) < 0) {
2676                 printf("%d: Error with start call\n", __LINE__);
2677                 return -1;
2678         }
2679
2680         /* Enqueue 3 packets to the rx port */
2681         for (i = 0; i < 3; i++) {
2682                 struct rte_event ev;
2683                 mbufs[i] = rte_gen_arp(0, t->mbuf_pool);
2684                 if (!mbufs[i]) {
2685                         printf("%d: gen of pkt failed\n", __LINE__);
2686                         return -1;
2687                 }
2688
2689                 ev.queue_id = t->qid[0];
2690                 ev.op = RTE_EVENT_OP_NEW;
2691                 ev.mbuf = mbufs[i];
2692                 mbufs[i]->seqn = MAGIC_SEQN + i;
2693
2694                 /* generate pkt and enqueue */
2695                 err = rte_event_enqueue_burst(evdev, t->port[rx_port], &ev, 1);
2696                 if (err != 1) {
2697                         printf("%d: Failed to enqueue pkt %u, retval = %u\n",
2698                                         __LINE__, i, err);
2699                         return -1;
2700                 }
2701         }
2702
2703         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2704
2705         /* use extra slot to make logic in loops easier */
2706         struct rte_event deq_ev[w3_port + 1];
2707
2708         /* Dequeue the 3 packets, one from each worker port */
2709         for (i = w1_port; i <= w3_port; i++) {
2710                 deq_pkts = rte_event_dequeue_burst(evdev, t->port[i],
2711                                 &deq_ev[i], 1, 0);
2712                 if (deq_pkts != 1) {
2713                         printf("%d: Failed to deq\n", __LINE__);
2714                         rte_event_dev_dump(evdev, stdout);
2715                         return -1;
2716                 }
2717         }
2718
2719         /* Enqueue each packet in reverse order, flushing after each one */
2720         for (i = w3_port; i >= w1_port; i--) {
2721
2722                 deq_ev[i].op = RTE_EVENT_OP_FORWARD;
2723                 deq_ev[i].queue_id = t->qid[1];
2724                 err = rte_event_enqueue_burst(evdev, t->port[i], &deq_ev[i], 1);
2725                 if (err != 1) {
2726                         printf("%d: Failed to enqueue\n", __LINE__);
2727                         return -1;
2728                 }
2729         }
2730         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2731
2732         /* dequeue from the tx ports, we should get 3 packets */
2733         deq_pkts = rte_event_dequeue_burst(evdev, t->port[tx_port], deq_ev,
2734                         3, 0);
2735
2736         /* Check to see if we've got all 3 packets */
2737         if (deq_pkts != 3) {
2738                 printf("%d: expected 3 pkts at tx port got %d from port %d\n",
2739                         __LINE__, deq_pkts, tx_port);
2740                 rte_event_dev_dump(evdev, stdout);
2741                 return 1;
2742         }
2743
2744         /* Check to see if the sequence numbers are in expected order */
2745         if (check_order) {
2746                 for (j = 0 ; j < deq_pkts ; j++) {
2747                         if (deq_ev[j].mbuf->seqn != MAGIC_SEQN + j) {
2748                                 printf(
2749                                         "%d: Incorrect sequence number(%d) from port %d\n",
2750                                         __LINE__, mbufs_out[j]->seqn, tx_port);
2751                                 return -1;
2752                         }
2753                 }
2754         }
2755
2756         /* Destroy the instance */
2757         cleanup(t);
2758         return 0;
2759 }
2760
2761 static int
2762 ordered_basic(struct test *t)
2763 {
2764         return parallel_basic(t, 1);
2765 }
2766
2767 static int
2768 unordered_basic(struct test *t)
2769 {
2770         return parallel_basic(t, 0);
2771 }
2772
2773 static int
2774 holb(struct test *t) /* test to check we avoid basic head-of-line blocking */
2775 {
2776         const struct rte_event new_ev = {
2777                         .op = RTE_EVENT_OP_NEW
2778                         /* all other fields zero */
2779         };
2780         struct rte_event ev = new_ev;
2781         unsigned int rx_port = 0; /* port we get the first flow on */
2782         char rx_port_used_stat[64];
2783         char rx_port_free_stat[64];
2784         char other_port_used_stat[64];
2785
2786         if (init(t, 1, 2) < 0 ||
2787                         create_ports(t, 2) < 0 ||
2788                         create_atomic_qids(t, 1) < 0) {
2789                 printf("%d: Error initializing device\n", __LINE__);
2790                 return -1;
2791         }
2792         int nb_links = rte_event_port_link(evdev, t->port[1], NULL, NULL, 0);
2793         if (rte_event_port_link(evdev, t->port[0], NULL, NULL, 0) != 1 ||
2794                         nb_links != 1) {
2795                 printf("%d: Error links queue to ports\n", __LINE__);
2796                 goto err;
2797         }
2798         if (rte_event_dev_start(evdev) < 0) {
2799                 printf("%d: Error with start call\n", __LINE__);
2800                 goto err;
2801         }
2802
2803         /* send one packet and see where it goes, port 0 or 1 */
2804         if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2805                 printf("%d: Error doing first enqueue\n", __LINE__);
2806                 goto err;
2807         }
2808         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2809
2810         if (rte_event_dev_xstats_by_name_get(evdev, "port_0_cq_ring_used", NULL)
2811                         != 1)
2812                 rx_port = 1;
2813
2814         snprintf(rx_port_used_stat, sizeof(rx_port_used_stat),
2815                         "port_%u_cq_ring_used", rx_port);
2816         snprintf(rx_port_free_stat, sizeof(rx_port_free_stat),
2817                         "port_%u_cq_ring_free", rx_port);
2818         snprintf(other_port_used_stat, sizeof(other_port_used_stat),
2819                         "port_%u_cq_ring_used", rx_port ^ 1);
2820         if (rte_event_dev_xstats_by_name_get(evdev, rx_port_used_stat, NULL)
2821                         != 1) {
2822                 printf("%d: Error, first event not scheduled\n", __LINE__);
2823                 goto err;
2824         }
2825
2826         /* now fill up the rx port's queue with one flow to cause HOLB */
2827         do {
2828                 ev = new_ev;
2829                 if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2830                         printf("%d: Error with enqueue\n", __LINE__);
2831                         goto err;
2832                 }
2833                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
2834         } while (rte_event_dev_xstats_by_name_get(evdev,
2835                                 rx_port_free_stat, NULL) != 0);
2836
2837         /* one more packet, which needs to stay in IQ - i.e. HOLB */
2838         ev = new_ev;
2839         if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2840                 printf("%d: Error with enqueue\n", __LINE__);
2841                 goto err;
2842         }
2843         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2844
2845         /* check that the other port still has an empty CQ */
2846         if (rte_event_dev_xstats_by_name_get(evdev, other_port_used_stat, NULL)
2847                         != 0) {
2848                 printf("%d: Error, second port CQ is not empty\n", __LINE__);
2849                 goto err;
2850         }
2851         /* check IQ now has one packet */
2852         if (rte_event_dev_xstats_by_name_get(evdev, "qid_0_iq_0_used", NULL)
2853                         != 1) {
2854                 printf("%d: Error, QID does not have exactly 1 packet\n",
2855                         __LINE__);
2856                 goto err;
2857         }
2858
2859         /* send another flow, which should pass the other IQ entry */
2860         ev = new_ev;
2861         ev.flow_id = 1;
2862         if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2863                 printf("%d: Error with enqueue\n", __LINE__);
2864                 goto err;
2865         }
2866         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2867
2868         if (rte_event_dev_xstats_by_name_get(evdev, other_port_used_stat, NULL)
2869                         != 1) {
2870                 printf("%d: Error, second flow did not pass out first\n",
2871                         __LINE__);
2872                 goto err;
2873         }
2874
2875         if (rte_event_dev_xstats_by_name_get(evdev, "qid_0_iq_0_used", NULL)
2876                         != 1) {
2877                 printf("%d: Error, QID does not have exactly 1 packet\n",
2878                         __LINE__);
2879                 goto err;
2880         }
2881         cleanup(t);
2882         return 0;
2883 err:
2884         rte_event_dev_dump(evdev, stdout);
2885         cleanup(t);
2886         return -1;
2887 }
2888
2889 static void
2890 flush(uint8_t dev_id __rte_unused, struct rte_event event, void *arg)
2891 {
2892         *((uint8_t *) arg) += (event.u64 == 0xCA11BACC) ? 1 : 0;
2893 }
2894
2895 static int
2896 dev_stop_flush(struct test *t) /* test to check we can properly flush events */
2897 {
2898         const struct rte_event new_ev = {
2899                 .op = RTE_EVENT_OP_NEW,
2900                 .u64 = 0xCA11BACC,
2901                 .queue_id = 0
2902         };
2903         struct rte_event ev = new_ev;
2904         uint8_t count = 0;
2905         int i;
2906
2907         if (init(t, 1, 1) < 0 ||
2908             create_ports(t, 1) < 0 ||
2909             create_atomic_qids(t, 1) < 0) {
2910                 printf("%d: Error initializing device\n", __LINE__);
2911                 return -1;
2912         }
2913
2914         /* Link the queue so *_start() doesn't error out */
2915         if (rte_event_port_link(evdev, t->port[0], NULL, NULL, 0) != 1) {
2916                 printf("%d: Error linking queue to port\n", __LINE__);
2917                 goto err;
2918         }
2919
2920         if (rte_event_dev_start(evdev) < 0) {
2921                 printf("%d: Error with start call\n", __LINE__);
2922                 goto err;
2923         }
2924
2925         for (i = 0; i < DEQUEUE_DEPTH + 1; i++) {
2926                 if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
2927                         printf("%d: Error enqueuing events\n", __LINE__);
2928                         goto err;
2929                 }
2930         }
2931
2932         /* Schedule the events from the port to the IQ. At least one event
2933          * should be remaining in the queue.
2934          */
2935         rte_service_run_iter_on_app_lcore(t->service_id, 1);
2936
2937         if (rte_event_dev_stop_flush_callback_register(evdev, flush, &count)) {
2938                 printf("%d: Error installing the flush callback\n", __LINE__);
2939                 goto err;
2940         }
2941
2942         cleanup(t);
2943
2944         if (count == 0) {
2945                 printf("%d: Error executing the flush callback\n", __LINE__);
2946                 goto err;
2947         }
2948
2949         if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
2950                 printf("%d: Error uninstalling the flush callback\n", __LINE__);
2951                 goto err;
2952         }
2953
2954         return 0;
2955 err:
2956         rte_event_dev_dump(evdev, stdout);
2957         cleanup(t);
2958         return -1;
2959 }
2960
2961 static int
2962 worker_loopback_worker_fn(void *arg)
2963 {
2964         struct test *t = arg;
2965         uint8_t port = t->port[1];
2966         int count = 0;
2967         int enqd;
2968
2969         /*
2970          * Takes packets from the input port and then loops them back through
2971          * the Eventdev. Each packet gets looped through QIDs 0-8, 16 times
2972          * so each packet goes through 8*16 = 128 times.
2973          */
2974         printf("%d: \tWorker function started\n", __LINE__);
2975         while (count < NUM_PACKETS) {
2976 #define BURST_SIZE 32
2977                 struct rte_event ev[BURST_SIZE];
2978                 uint16_t i, nb_rx = rte_event_dequeue_burst(evdev, port, ev,
2979                                 BURST_SIZE, 0);
2980                 if (nb_rx == 0) {
2981                         rte_pause();
2982                         continue;
2983                 }
2984
2985                 for (i = 0; i < nb_rx; i++) {
2986                         ev[i].queue_id++;
2987                         if (ev[i].queue_id != 8) {
2988                                 ev[i].op = RTE_EVENT_OP_FORWARD;
2989                                 enqd = rte_event_enqueue_burst(evdev, port,
2990                                                 &ev[i], 1);
2991                                 if (enqd != 1) {
2992                                         printf("%d: Can't enqueue FWD!!\n",
2993                                                         __LINE__);
2994                                         return -1;
2995                                 }
2996                                 continue;
2997                         }
2998
2999                         ev[i].queue_id = 0;
3000                         (*counter_field(ev[i].mbuf))++;
3001                         if (*counter_field(ev[i].mbuf) != 16) {
3002                                 ev[i].op = RTE_EVENT_OP_FORWARD;
3003                                 enqd = rte_event_enqueue_burst(evdev, port,
3004                                                 &ev[i], 1);
3005                                 if (enqd != 1) {
3006                                         printf("%d: Can't enqueue FWD!!\n",
3007                                                         __LINE__);
3008                                         return -1;
3009                                 }
3010                                 continue;
3011                         }
3012                         /* we have hit 16 iterations through system - drop */
3013                         rte_pktmbuf_free(ev[i].mbuf);
3014                         count++;
3015                         ev[i].op = RTE_EVENT_OP_RELEASE;
3016                         enqd = rte_event_enqueue_burst(evdev, port, &ev[i], 1);
3017                         if (enqd != 1) {
3018                                 printf("%d drop enqueue failed\n", __LINE__);
3019                                 return -1;
3020                         }
3021                 }
3022         }
3023
3024         return 0;
3025 }
3026
3027 static int
3028 worker_loopback_producer_fn(void *arg)
3029 {
3030         struct test *t = arg;
3031         uint8_t port = t->port[0];
3032         uint64_t count = 0;
3033
3034         printf("%d: \tProducer function started\n", __LINE__);
3035         while (count < NUM_PACKETS) {
3036                 struct rte_mbuf *m = 0;
3037                 do {
3038                         m = rte_pktmbuf_alloc(t->mbuf_pool);
3039                 } while (m == NULL);
3040
3041                 *counter_field(m) = 0;
3042
3043                 struct rte_event ev = {
3044                                 .op = RTE_EVENT_OP_NEW,
3045                                 .queue_id = t->qid[0],
3046                                 .flow_id = (uintptr_t)m & 0xFFFF,
3047                                 .mbuf = m,
3048                 };
3049
3050                 if (rte_event_enqueue_burst(evdev, port, &ev, 1) != 1) {
3051                         while (rte_event_enqueue_burst(evdev, port, &ev, 1) !=
3052                                         1)
3053                                 rte_pause();
3054                 }
3055
3056                 count++;
3057         }
3058
3059         return 0;
3060 }
3061
3062 static int
3063 worker_loopback(struct test *t, uint8_t disable_implicit_release)
3064 {
3065         /* use a single producer core, and a worker core to see what happens
3066          * if the worker loops packets back multiple times
3067          */
3068         struct test_event_dev_stats stats;
3069         uint64_t print_cycles = 0, cycles = 0;
3070         uint64_t tx_pkts = 0;
3071         int err;
3072         int w_lcore, p_lcore;
3073
3074         static const struct rte_mbuf_dynfield counter_dynfield_desc = {
3075                 .name = "rte_event_sw_dynfield_selftest_counter",
3076                 .size = sizeof(counter_dynfield_t),
3077                 .align = __alignof__(counter_dynfield_t),
3078         };
3079         counter_dynfield_offset =
3080                 rte_mbuf_dynfield_register(&counter_dynfield_desc);
3081         if (counter_dynfield_offset < 0) {
3082                 printf("Error registering mbuf field\n");
3083                 return -rte_errno;
3084         }
3085
3086         if (init(t, 8, 2) < 0 ||
3087                         create_atomic_qids(t, 8) < 0) {
3088                 printf("%d: Error initializing device\n", __LINE__);
3089                 return -1;
3090         }
3091
3092         /* RX with low max events */
3093         static struct rte_event_port_conf conf = {
3094                         .dequeue_depth = 32,
3095                         .enqueue_depth = 64,
3096         };
3097         /* beware: this cannot be initialized in the static above as it would
3098          * only be initialized once - and this needs to be set for multiple runs
3099          */
3100         conf.new_event_threshold = 512;
3101         conf.event_port_cfg = disable_implicit_release ?
3102                 RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL : 0;
3103
3104         if (rte_event_port_setup(evdev, 0, &conf) < 0) {
3105                 printf("Error setting up RX port\n");
3106                 return -1;
3107         }
3108         t->port[0] = 0;
3109         /* TX with higher max events */
3110         conf.new_event_threshold = 4096;
3111         if (rte_event_port_setup(evdev, 1, &conf) < 0) {
3112                 printf("Error setting up TX port\n");
3113                 return -1;
3114         }
3115         t->port[1] = 1;
3116
3117         /* CQ mapping to QID */
3118         err = rte_event_port_link(evdev, t->port[1], NULL, NULL, 0);
3119         if (err != 8) { /* should have mapped all queues*/
3120                 printf("%d: error mapping port 2 to all qids\n", __LINE__);
3121                 return -1;
3122         }
3123
3124         if (rte_event_dev_start(evdev) < 0) {
3125                 printf("%d: Error with start call\n", __LINE__);
3126                 return -1;
3127         }
3128
3129         p_lcore = rte_get_next_lcore(
3130                         /* start core */ -1,
3131                         /* skip main */ 1,
3132                         /* wrap */ 0);
3133         w_lcore = rte_get_next_lcore(p_lcore, 1, 0);
3134
3135         rte_eal_remote_launch(worker_loopback_producer_fn, t, p_lcore);
3136         rte_eal_remote_launch(worker_loopback_worker_fn, t, w_lcore);
3137
3138         print_cycles = cycles = rte_get_timer_cycles();
3139         while (rte_eal_get_lcore_state(p_lcore) != FINISHED ||
3140                         rte_eal_get_lcore_state(w_lcore) != FINISHED) {
3141
3142                 rte_service_run_iter_on_app_lcore(t->service_id, 1);
3143
3144                 uint64_t new_cycles = rte_get_timer_cycles();
3145
3146                 if (new_cycles - print_cycles > rte_get_timer_hz()) {
3147                         test_event_dev_stats_get(evdev, &stats);
3148                         printf(
3149                                 "%d: \tSched Rx = %"PRIu64", Tx = %"PRIu64"\n",
3150                                 __LINE__, stats.rx_pkts, stats.tx_pkts);
3151
3152                         print_cycles = new_cycles;
3153                 }
3154                 if (new_cycles - cycles > rte_get_timer_hz() * 3) {
3155                         test_event_dev_stats_get(evdev, &stats);
3156                         if (stats.tx_pkts == tx_pkts) {
3157                                 rte_event_dev_dump(evdev, stdout);
3158                                 printf("Dumping xstats:\n");
3159                                 xstats_print();
3160                                 printf(
3161                                         "%d: No schedules for seconds, deadlock\n",
3162                                         __LINE__);
3163                                 return -1;
3164                         }
3165                         tx_pkts = stats.tx_pkts;
3166                         cycles = new_cycles;
3167                 }
3168         }
3169         rte_service_run_iter_on_app_lcore(t->service_id, 1);
3170         /* ensure all completions are flushed */
3171
3172         rte_eal_mp_wait_lcore();
3173
3174         cleanup(t);
3175         return 0;
3176 }
3177
3178 static struct rte_mempool *eventdev_func_mempool;
3179
3180 int
3181 test_sw_eventdev(void)
3182 {
3183         struct test *t;
3184         int ret;
3185
3186         t = malloc(sizeof(struct test));
3187         if (t == NULL)
3188                 return -1;
3189         /* manually initialize the op, older gcc's complain on static
3190          * initialization of struct elements that are a bitfield.
3191          */
3192         release_ev.op = RTE_EVENT_OP_RELEASE;
3193
3194         const char *eventdev_name = "event_sw";
3195         evdev = rte_event_dev_get_dev_id(eventdev_name);
3196         if (evdev < 0) {
3197                 printf("%d: Eventdev %s not found - creating.\n",
3198                                 __LINE__, eventdev_name);
3199                 if (rte_vdev_init(eventdev_name, NULL) < 0) {
3200                         printf("Error creating eventdev\n");
3201                         goto test_fail;
3202                 }
3203                 evdev = rte_event_dev_get_dev_id(eventdev_name);
3204                 if (evdev < 0) {
3205                         printf("Error finding newly created eventdev\n");
3206                         goto test_fail;
3207                 }
3208         }
3209
3210         if (rte_event_dev_service_id_get(evdev, &t->service_id) < 0) {
3211                 printf("Failed to get service ID for software event dev\n");
3212                 goto test_fail;
3213         }
3214
3215         rte_service_runstate_set(t->service_id, 1);
3216         rte_service_set_runstate_mapped_check(t->service_id, 0);
3217
3218         /* Only create mbuf pool once, reuse for each test run */
3219         if (!eventdev_func_mempool) {
3220                 eventdev_func_mempool = rte_pktmbuf_pool_create(
3221                                 "EVENTDEV_SW_SA_MBUF_POOL",
3222                                 (1<<12), /* 4k buffers */
3223                                 32 /*MBUF_CACHE_SIZE*/,
3224                                 0,
3225                                 512, /* use very small mbufs */
3226                                 rte_socket_id());
3227                 if (!eventdev_func_mempool) {
3228                         printf("ERROR creating mempool\n");
3229                         goto test_fail;
3230                 }
3231         }
3232         t->mbuf_pool = eventdev_func_mempool;
3233         printf("*** Running Single Directed Packet test...\n");
3234         ret = test_single_directed_packet(t);
3235         if (ret != 0) {
3236                 printf("ERROR - Single Directed Packet test FAILED.\n");
3237                 goto test_fail;
3238         }
3239         printf("*** Running Directed Forward Credit test...\n");
3240         ret = test_directed_forward_credits(t);
3241         if (ret != 0) {
3242                 printf("ERROR - Directed Forward Credit test FAILED.\n");
3243                 goto test_fail;
3244         }
3245         printf("*** Running Single Load Balanced Packet test...\n");
3246         ret = single_packet(t);
3247         if (ret != 0) {
3248                 printf("ERROR - Single Packet test FAILED.\n");
3249                 goto test_fail;
3250         }
3251         printf("*** Running Unordered Basic test...\n");
3252         ret = unordered_basic(t);
3253         if (ret != 0) {
3254                 printf("ERROR -  Unordered Basic test FAILED.\n");
3255                 goto test_fail;
3256         }
3257         printf("*** Running Ordered Basic test...\n");
3258         ret = ordered_basic(t);
3259         if (ret != 0) {
3260                 printf("ERROR -  Ordered Basic test FAILED.\n");
3261                 goto test_fail;
3262         }
3263         printf("*** Running Burst Packets test...\n");
3264         ret = burst_packets(t);
3265         if (ret != 0) {
3266                 printf("ERROR - Burst Packets test FAILED.\n");
3267                 goto test_fail;
3268         }
3269         printf("*** Running Load Balancing test...\n");
3270         ret = load_balancing(t);
3271         if (ret != 0) {
3272                 printf("ERROR - Load Balancing test FAILED.\n");
3273                 goto test_fail;
3274         }
3275         printf("*** Running Prioritized Directed test...\n");
3276         ret = test_priority_directed(t);
3277         if (ret != 0) {
3278                 printf("ERROR - Prioritized Directed test FAILED.\n");
3279                 goto test_fail;
3280         }
3281         printf("*** Running Prioritized Atomic test...\n");
3282         ret = test_priority_atomic(t);
3283         if (ret != 0) {
3284                 printf("ERROR - Prioritized Atomic test FAILED.\n");
3285                 goto test_fail;
3286         }
3287
3288         printf("*** Running Prioritized Ordered test...\n");
3289         ret = test_priority_ordered(t);
3290         if (ret != 0) {
3291                 printf("ERROR - Prioritized Ordered test FAILED.\n");
3292                 goto test_fail;
3293         }
3294         printf("*** Running Prioritized Unordered test...\n");
3295         ret = test_priority_unordered(t);
3296         if (ret != 0) {
3297                 printf("ERROR - Prioritized Unordered test FAILED.\n");
3298                 goto test_fail;
3299         }
3300         printf("*** Running Invalid QID test...\n");
3301         ret = invalid_qid(t);
3302         if (ret != 0) {
3303                 printf("ERROR - Invalid QID test FAILED.\n");
3304                 goto test_fail;
3305         }
3306         printf("*** Running Load Balancing History test...\n");
3307         ret = load_balancing_history(t);
3308         if (ret != 0) {
3309                 printf("ERROR - Load Balancing History test FAILED.\n");
3310                 goto test_fail;
3311         }
3312         printf("*** Running Inflight Count test...\n");
3313         ret = inflight_counts(t);
3314         if (ret != 0) {
3315                 printf("ERROR - Inflight Count test FAILED.\n");
3316                 goto test_fail;
3317         }
3318         printf("*** Running Abuse Inflights test...\n");
3319         ret = abuse_inflights(t);
3320         if (ret != 0) {
3321                 printf("ERROR - Abuse Inflights test FAILED.\n");
3322                 goto test_fail;
3323         }
3324         printf("*** Running XStats test...\n");
3325         ret = xstats_tests(t);
3326         if (ret != 0) {
3327                 printf("ERROR - XStats test FAILED.\n");
3328                 goto test_fail;
3329         }
3330         printf("*** Running XStats ID Reset test...\n");
3331         ret = xstats_id_reset_tests(t);
3332         if (ret != 0) {
3333                 printf("ERROR - XStats ID Reset test FAILED.\n");
3334                 goto test_fail;
3335         }
3336         printf("*** Running XStats Brute Force test...\n");
3337         ret = xstats_brute_force(t);
3338         if (ret != 0) {
3339                 printf("ERROR - XStats Brute Force test FAILED.\n");
3340                 goto test_fail;
3341         }
3342         printf("*** Running XStats ID Abuse test...\n");
3343         ret = xstats_id_abuse_tests(t);
3344         if (ret != 0) {
3345                 printf("ERROR - XStats ID Abuse test FAILED.\n");
3346                 goto test_fail;
3347         }
3348         printf("*** Running QID Priority test...\n");
3349         ret = qid_priorities(t);
3350         if (ret != 0) {
3351                 printf("ERROR - QID Priority test FAILED.\n");
3352                 goto test_fail;
3353         }
3354         printf("*** Running Unlink-in-progress test...\n");
3355         ret = unlink_in_progress(t);
3356         if (ret != 0) {
3357                 printf("ERROR - Unlink in progress test FAILED.\n");
3358                 goto test_fail;
3359         }
3360         printf("*** Running Ordered Reconfigure test...\n");
3361         ret = ordered_reconfigure(t);
3362         if (ret != 0) {
3363                 printf("ERROR - Ordered Reconfigure test FAILED.\n");
3364                 goto test_fail;
3365         }
3366         printf("*** Running Port LB Single Reconfig test...\n");
3367         ret = port_single_lb_reconfig(t);
3368         if (ret != 0) {
3369                 printf("ERROR - Port LB Single Reconfig test FAILED.\n");
3370                 goto test_fail;
3371         }
3372         printf("*** Running Port Reconfig Credits test...\n");
3373         ret = port_reconfig_credits(t);
3374         if (ret != 0) {
3375                 printf("ERROR - Port Reconfig Credits Reset test FAILED.\n");
3376                 goto test_fail;
3377         }
3378         printf("*** Running Head-of-line-blocking test...\n");
3379         ret = holb(t);
3380         if (ret != 0) {
3381                 printf("ERROR - Head-of-line-blocking test FAILED.\n");
3382                 goto test_fail;
3383         }
3384         printf("*** Running Stop Flush test...\n");
3385         ret = dev_stop_flush(t);
3386         if (ret != 0) {
3387                 printf("ERROR - Stop Flush test FAILED.\n");
3388                 goto test_fail;
3389         }
3390         if (rte_lcore_count() >= 3) {
3391                 printf("*** Running Worker loopback test...\n");
3392                 ret = worker_loopback(t, 0);
3393                 if (ret != 0) {
3394                         printf("ERROR - Worker loopback test FAILED.\n");
3395                         return ret;
3396                 }
3397
3398                 printf("*** Running Worker loopback test (implicit release disabled)...\n");
3399                 ret = worker_loopback(t, 1);
3400                 if (ret != 0) {
3401                         printf("ERROR - Worker loopback test FAILED.\n");
3402                         goto test_fail;
3403                 }
3404         } else {
3405                 printf("### Not enough cores for worker loopback tests.\n");
3406                 printf("### Need at least 3 cores for the tests.\n");
3407         }
3408
3409         /*
3410          * Free test instance, leaving mempool initialized, and a pointer to it
3411          * in static eventdev_func_mempool, as it is re-used on re-runs
3412          */
3413         free(t);
3414
3415         printf("SW Eventdev Selftest Successful.\n");
3416         return 0;
3417 test_fail:
3418         free(t);
3419         printf("SW Eventdev Selftest Failed.\n");
3420         return -1;
3421 }