drivers/event/cnxk/cnxk_eventdev_selftest.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_cycles.h>
   7 #include <rte_debug.h>
   8 #include <rte_eal.h>
   9 #include <rte_ethdev.h>
  10 #include <rte_eventdev.h>
  11 #include <rte_hexdump.h>
  12 #include <rte_launch.h>
  13 #include <rte_lcore.h>
  14 #include <rte_malloc.h>
  15 #include <rte_mbuf.h>
  16 #include <rte_memcpy.h>
  17 #include <rte_per_lcore.h>
  18 #include <rte_random.h>
  19 #include <rte_test.h>
  20
  21 #include "cnxk_eventdev.h"
  22
  23 #define NUM_PACKETS (1024)
  24 #define MAX_EVENTS  (1024)
  25 #define MAX_STAGES  (255)
  26
  27 #define CNXK_TEST_RUN(setup, teardown, test)                                   \
  28         cnxk_test_run(setup, teardown, test, #test)
  29
  30 static int total;
  31 static int passed;
  32 static int failed;
  33 static int unsupported;
  34
  35 static int evdev;
  36 static struct rte_mempool *eventdev_test_mempool;
  37
  38 struct event_attr {
  39         uint32_t flow_id;
  40         uint8_t event_type;
  41         uint8_t sub_event_type;
  42         uint8_t sched_type;
  43         uint8_t queue;
  44         uint8_t port;
  45 };
  46
  47 static uint32_t seqn_list_index;
  48 static int seqn_list[NUM_PACKETS];
  49
  50 static inline void
  51 seqn_list_init(void)
  52 {
  53         RTE_BUILD_BUG_ON(NUM_PACKETS < MAX_EVENTS);
  54         memset(seqn_list, 0, sizeof(seqn_list));
  55         seqn_list_index = 0;
  56 }
  57
  58 static inline int
  59 seqn_list_update(int val)
  60 {
  61         if (seqn_list_index >= NUM_PACKETS)
  62                 return -1;
  63
  64         seqn_list[seqn_list_index++] = val;
  65         rte_atomic_thread_fence(__ATOMIC_RELEASE);
  66         return 0;
  67 }
  68
  69 static inline int
  70 seqn_list_check(int limit)
  71 {
  72         int i;
  73
  74         for (i = 0; i < limit; i++) {
  75                 if (seqn_list[i] != i) {
  76                         plt_err("Seqn mismatch %d %d", seqn_list[i], i);
  77                         return -1;
  78                 }
  79         }
  80         return 0;
  81 }
  82
  83 struct test_core_param {
  84         uint32_t *total_events;
  85         uint64_t dequeue_tmo_ticks;
  86         uint8_t port;
  87         uint8_t sched_type;
  88 };
  89
  90 static int
  91 testsuite_setup(const char *eventdev_name)
  92 {
  93         evdev = rte_event_dev_get_dev_id(eventdev_name);
  94         if (evdev < 0) {
  95                 plt_err("%d: Eventdev %s not found", __LINE__, eventdev_name);
  96                 return -1;
  97         }
  98         return 0;
  99 }
 100
 101 static void
 102 testsuite_teardown(void)
 103 {
 104         rte_event_dev_close(evdev);
 105         total = 0;
 106         passed = 0;
 107         failed = 0;
 108         unsupported = 0;
 109 }
 110
 111 static inline void
 112 devconf_set_default_sane_values(struct rte_event_dev_config *dev_conf,
 113                                 struct rte_event_dev_info *info)
 114 {
 115         memset(dev_conf, 0, sizeof(struct rte_event_dev_config));
 116         dev_conf->dequeue_timeout_ns = info->min_dequeue_timeout_ns;
 117         dev_conf->nb_event_ports = info->max_event_ports;
 118         dev_conf->nb_event_queues = info->max_event_queues;
 119         dev_conf->nb_event_queue_flows = info->max_event_queue_flows;
 120         dev_conf->nb_event_port_dequeue_depth =
 121                 info->max_event_port_dequeue_depth;
 122         dev_conf->nb_event_port_enqueue_depth =
 123                 info->max_event_port_enqueue_depth;
 124         dev_conf->nb_event_port_enqueue_depth =
 125                 info->max_event_port_enqueue_depth;
 126         dev_conf->nb_events_limit = info->max_num_events;
 127 }
 128
 129 enum {
 130         TEST_EVENTDEV_SETUP_DEFAULT,
 131         TEST_EVENTDEV_SETUP_PRIORITY,
 132         TEST_EVENTDEV_SETUP_DEQUEUE_TIMEOUT,
 133 };
 134
 135 static inline int
 136 _eventdev_setup(int mode)
 137 {
 138         const char *pool_name = "evdev_cnxk_test_pool";
 139         struct rte_event_dev_config dev_conf;
 140         struct rte_event_dev_info info;
 141         int i, ret;
 142
 143         /* Create and destrory pool for each test case to make it standalone */
 144         eventdev_test_mempool = rte_pktmbuf_pool_create(
 145                 pool_name, MAX_EVENTS, 0, 0, 512, rte_socket_id());
 146         if (!eventdev_test_mempool) {
 147                 plt_err("ERROR creating mempool");
 148                 return -1;
 149         }
 150
 151         ret = rte_event_dev_info_get(evdev, &info);
 152         RTE_TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info");
 153
 154         devconf_set_default_sane_values(&dev_conf, &info);
 155         if (mode == TEST_EVENTDEV_SETUP_DEQUEUE_TIMEOUT)
 156                 dev_conf.event_dev_cfg |= RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
 157
 158         ret = rte_event_dev_configure(evdev, &dev_conf);
 159         RTE_TEST_ASSERT_SUCCESS(ret, "Failed to configure eventdev");
 160
 161         uint32_t queue_count;
 162         RTE_TEST_ASSERT_SUCCESS(
 163                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 164                                        &queue_count),
 165                 "Queue count get failed");
 166
 167         if (mode == TEST_EVENTDEV_SETUP_PRIORITY) {
 168                 if (queue_count > 8)
 169                         queue_count = 8;
 170
 171                 /* Configure event queues(0 to n) with
 172                  * RTE_EVENT_DEV_PRIORITY_HIGHEST to
 173                  * RTE_EVENT_DEV_PRIORITY_LOWEST
 174                  */
 175                 uint8_t step =
 176                         (RTE_EVENT_DEV_PRIORITY_LOWEST + 1) / queue_count;
 177                 for (i = 0; i < (int)queue_count; i++) {
 178                         struct rte_event_queue_conf queue_conf;
 179
 180                         ret = rte_event_queue_default_conf_get(evdev, i,
 181                                                                &queue_conf);
 182                         RTE_TEST_ASSERT_SUCCESS(ret, "Failed to get def_conf%d",
 183                                                 i);
 184                         queue_conf.priority = i * step;
 185                         ret = rte_event_queue_setup(evdev, i, &queue_conf);
 186                         RTE_TEST_ASSERT_SUCCESS(ret, "Failed to setup queue=%d",
 187                                                 i);
 188                 }
 189
 190         } else {
 191                 /* Configure event queues with default priority */
 192                 for (i = 0; i < (int)queue_count; i++) {
 193                         ret = rte_event_queue_setup(evdev, i, NULL);
 194                         RTE_TEST_ASSERT_SUCCESS(ret, "Failed to setup queue=%d",
 195                                                 i);
 196                 }
 197         }
 198         /* Configure event ports */
 199         uint32_t port_count;
 200         RTE_TEST_ASSERT_SUCCESS(
 201                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_PORT_COUNT,
 202                                        &port_count),
 203                 "Port count get failed");
 204         for (i = 0; i < (int)port_count; i++) {
 205                 ret = rte_event_port_setup(evdev, i, NULL);
 206                 RTE_TEST_ASSERT_SUCCESS(ret, "Failed to setup port=%d", i);
 207                 ret = rte_event_port_link(evdev, i, NULL, NULL, 0);
 208                 RTE_TEST_ASSERT(ret >= 0, "Failed to link all queues port=%d",
 209                                 i);
 210         }
 211
 212         ret = rte_event_dev_start(evdev);
 213         RTE_TEST_ASSERT_SUCCESS(ret, "Failed to start device");
 214
 215         return 0;
 216 }
 217
 218 static inline int
 219 eventdev_setup(void)
 220 {
 221         return _eventdev_setup(TEST_EVENTDEV_SETUP_DEFAULT);
 222 }
 223
 224 static inline int
 225 eventdev_setup_priority(void)
 226 {
 227         return _eventdev_setup(TEST_EVENTDEV_SETUP_PRIORITY);
 228 }
 229
 230 static inline int
 231 eventdev_setup_dequeue_timeout(void)
 232 {
 233         return _eventdev_setup(TEST_EVENTDEV_SETUP_DEQUEUE_TIMEOUT);
 234 }
 235
 236 static inline void
 237 eventdev_teardown(void)
 238 {
 239         rte_event_dev_stop(evdev);
 240         rte_mempool_free(eventdev_test_mempool);
 241 }
 242
 243 static inline void
 244 update_event_and_validation_attr(struct rte_mbuf *m, struct rte_event *ev,
 245                                  uint32_t flow_id, uint8_t event_type,
 246                                  uint8_t sub_event_type, uint8_t sched_type,
 247                                  uint8_t queue, uint8_t port)
 248 {
 249         struct event_attr *attr;
 250
 251         /* Store the event attributes in mbuf for future reference */
 252         attr = rte_pktmbuf_mtod(m, struct event_attr *);
 253         attr->flow_id = flow_id;
 254         attr->event_type = event_type;
 255         attr->sub_event_type = sub_event_type;
 256         attr->sched_type = sched_type;
 257         attr->queue = queue;
 258         attr->port = port;
 259
 260         ev->flow_id = flow_id;
 261         ev->sub_event_type = sub_event_type;
 262         ev->event_type = event_type;
 263         /* Inject the new event */
 264         ev->op = RTE_EVENT_OP_NEW;
 265         ev->sched_type = sched_type;
 266         ev->queue_id = queue;
 267         ev->mbuf = m;
 268 }
 269
 270 static inline int
 271 inject_events(uint32_t flow_id, uint8_t event_type, uint8_t sub_event_type,
 272               uint8_t sched_type, uint8_t queue, uint8_t port,
 273               unsigned int events)
 274 {
 275         struct rte_mbuf *m;
 276         unsigned int i;
 277
 278         for (i = 0; i < events; i++) {
 279                 struct rte_event ev = {.event = 0, .u64 = 0};
 280
 281                 m = rte_pktmbuf_alloc(eventdev_test_mempool);
 282                 RTE_TEST_ASSERT_NOT_NULL(m, "mempool alloc failed");
 283
 284                 *rte_event_pmd_selftest_seqn(m) = i;
 285                 update_event_and_validation_attr(m, &ev, flow_id, event_type,
 286                                                  sub_event_type, sched_type,
 287                                                  queue, port);
 288                 rte_event_enqueue_burst(evdev, port, &ev, 1);
 289         }
 290         return 0;
 291 }
 292
 293 static inline int
 294 check_excess_events(uint8_t port)
 295 {
 296         uint16_t valid_event;
 297         struct rte_event ev;
 298         int i;
 299
 300         /* Check for excess events, try for a few times and exit */
 301         for (i = 0; i < 32; i++) {
 302                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0);
 303
 304                 RTE_TEST_ASSERT_SUCCESS(valid_event,
 305                                         "Unexpected valid event=%d",
 306                                         *rte_event_pmd_selftest_seqn(ev.mbuf));
 307         }
 308         return 0;
 309 }
 310
 311 static inline int
 312 generate_random_events(const unsigned int total_events)
 313 {
 314         struct rte_event_dev_info info;
 315         uint32_t queue_count;
 316         unsigned int i;
 317         int ret;
 318
 319         RTE_TEST_ASSERT_SUCCESS(
 320                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 321                                        &queue_count),
 322                 "Queue count get failed");
 323
 324         ret = rte_event_dev_info_get(evdev, &info);
 325         RTE_TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info");
 326         for (i = 0; i < total_events; i++) {
 327                 ret = inject_events(
 328                         rte_rand() % info.max_event_queue_flows /*flow_id */,
 329                         RTE_EVENT_TYPE_CPU /* event_type */,
 330                         rte_rand() % 256 /* sub_event_type */,
 331                         rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1),
 332                         rte_rand() % queue_count /* queue */, 0 /* port */,
 333                         1 /* events */);
 334                 if (ret)
 335                         return -1;
 336         }
 337         return ret;
 338 }
 339
 340 static inline int
 341 validate_event(struct rte_event *ev)
 342 {
 343         struct event_attr *attr;
 344
 345         attr = rte_pktmbuf_mtod(ev->mbuf, struct event_attr *);
 346         RTE_TEST_ASSERT_EQUAL(attr->flow_id, ev->flow_id,
 347                               "flow_id mismatch enq=%d deq =%d", attr->flow_id,
 348                               ev->flow_id);
 349         RTE_TEST_ASSERT_EQUAL(attr->event_type, ev->event_type,
 350                               "event_type mismatch enq=%d deq =%d",
 351                               attr->event_type, ev->event_type);
 352         RTE_TEST_ASSERT_EQUAL(attr->sub_event_type, ev->sub_event_type,
 353                               "sub_event_type mismatch enq=%d deq =%d",
 354                               attr->sub_event_type, ev->sub_event_type);
 355         RTE_TEST_ASSERT_EQUAL(attr->sched_type, ev->sched_type,
 356                               "sched_type mismatch enq=%d deq =%d",
 357                               attr->sched_type, ev->sched_type);
 358         RTE_TEST_ASSERT_EQUAL(attr->queue, ev->queue_id,
 359                               "queue mismatch enq=%d deq =%d", attr->queue,
 360                               ev->queue_id);
 361         return 0;
 362 }
 363
 364 typedef int (*validate_event_cb)(uint32_t index, uint8_t port,
 365                                  struct rte_event *ev);
 366
 367 static inline int
 368 consume_events(uint8_t port, const uint32_t total_events, validate_event_cb fn)
 369 {
 370         uint32_t events = 0, forward_progress_cnt = 0, index = 0;
 371         uint16_t valid_event;
 372         struct rte_event ev;
 373         int ret;
 374
 375         while (1) {
 376                 if (++forward_progress_cnt > UINT16_MAX) {
 377                         plt_err("Detected deadlock");
 378                         return -1;
 379                 }
 380
 381                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0);
 382                 if (!valid_event)
 383                         continue;
 384
 385                 forward_progress_cnt = 0;
 386                 ret = validate_event(&ev);
 387                 if (ret)
 388                         return -1;
 389
 390                 if (fn != NULL) {
 391                         ret = fn(index, port, &ev);
 392                         RTE_TEST_ASSERT_SUCCESS(
 393                                 ret, "Failed to validate test specific event");
 394                 }
 395
 396                 ++index;
 397
 398                 rte_pktmbuf_free(ev.mbuf);
 399                 if (++events >= total_events)
 400                         break;
 401         }
 402
 403         return check_excess_events(port);
 404 }
 405
 406 static int
 407 validate_simple_enqdeq(uint32_t index, uint8_t port, struct rte_event *ev)
 408 {
 409         RTE_SET_USED(port);
 410         RTE_TEST_ASSERT_EQUAL(index, *rte_event_pmd_selftest_seqn(ev->mbuf),
 411                               "index=%d != seqn=%d", index,
 412                               *rte_event_pmd_selftest_seqn(ev->mbuf));
 413         return 0;
 414 }
 415
 416 static inline int
 417 test_simple_enqdeq(uint8_t sched_type)
 418 {
 419         int ret;
 420
 421         ret = inject_events(0 /*flow_id */, RTE_EVENT_TYPE_CPU /* event_type */,
 422                             0 /* sub_event_type */, sched_type, 0 /* queue */,
 423                             0 /* port */, MAX_EVENTS);
 424         if (ret)
 425                 return -1;
 426
 427         return consume_events(0 /* port */, MAX_EVENTS, validate_simple_enqdeq);
 428 }
 429
 430 static int
 431 test_simple_enqdeq_ordered(void)
 432 {
 433         return test_simple_enqdeq(RTE_SCHED_TYPE_ORDERED);
 434 }
 435
 436 static int
 437 test_simple_enqdeq_atomic(void)
 438 {
 439         return test_simple_enqdeq(RTE_SCHED_TYPE_ATOMIC);
 440 }
 441
 442 static int
 443 test_simple_enqdeq_parallel(void)
 444 {
 445         return test_simple_enqdeq(RTE_SCHED_TYPE_PARALLEL);
 446 }
 447
 448 /*
 449  * Generate a prescribed number of events and spread them across available
 450  * queues. On dequeue, using single event port(port 0) verify the enqueued
 451  * event attributes
 452  */
 453 static int
 454 test_multi_queue_enq_single_port_deq(void)
 455 {
 456         int ret;
 457
 458         ret = generate_random_events(MAX_EVENTS);
 459         if (ret)
 460                 return -1;
 461
 462         return consume_events(0 /* port */, MAX_EVENTS, NULL);
 463 }
 464
 465 /*
 466  * Inject 0..MAX_EVENTS events over 0..queue_count with modulus
 467  * operation
 468  *
 469  * For example, Inject 32 events over 0..7 queues
 470  * enqueue events 0, 8, 16, 24 in queue 0
 471  * enqueue events 1, 9, 17, 25 in queue 1
 472  * ..
 473  * ..
 474  * enqueue events 7, 15, 23, 31 in queue 7
 475  *
 476  * On dequeue, Validate the events comes in 0,8,16,24,1,9,17,25..,7,15,23,31
 477  * order from queue0(highest priority) to queue7(lowest_priority)
 478  */
 479 static int
 480 validate_queue_priority(uint32_t index, uint8_t port, struct rte_event *ev)
 481 {
 482         uint32_t queue_count;
 483
 484         RTE_TEST_ASSERT_SUCCESS(
 485                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 486                                        &queue_count),
 487                 "Queue count get failed");
 488         if (queue_count > 8)
 489                 queue_count = 8;
 490         uint32_t range = MAX_EVENTS / queue_count;
 491         uint32_t expected_val = (index % range) * queue_count;
 492
 493         expected_val += ev->queue_id;
 494         RTE_SET_USED(port);
 495         RTE_TEST_ASSERT_EQUAL(
 496                 *rte_event_pmd_selftest_seqn(ev->mbuf), expected_val,
 497                 "seqn=%d index=%d expected=%d range=%d nb_queues=%d max_event=%d",
 498                 *rte_event_pmd_selftest_seqn(ev->mbuf), index, expected_val,
 499                 range, queue_count, MAX_EVENTS);
 500         return 0;
 501 }
 502
 503 static int
 504 test_multi_queue_priority(void)
 505 {
 506         int i, max_evts_roundoff;
 507         /* See validate_queue_priority() comments for priority validate logic */
 508         uint32_t queue_count;
 509         struct rte_mbuf *m;
 510         uint8_t queue;
 511
 512         RTE_TEST_ASSERT_SUCCESS(
 513                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 514                                        &queue_count),
 515                 "Queue count get failed");
 516         if (queue_count > 8)
 517                 queue_count = 8;
 518         max_evts_roundoff = MAX_EVENTS / queue_count;
 519         max_evts_roundoff *= queue_count;
 520
 521         for (i = 0; i < max_evts_roundoff; i++) {
 522                 struct rte_event ev = {.event = 0, .u64 = 0};
 523
 524                 m = rte_pktmbuf_alloc(eventdev_test_mempool);
 525                 RTE_TEST_ASSERT_NOT_NULL(m, "mempool alloc failed");
 526
 527                 *rte_event_pmd_selftest_seqn(m) = i;
 528                 queue = i % queue_count;
 529                 update_event_and_validation_attr(m, &ev, 0, RTE_EVENT_TYPE_CPU,
 530                                                  0, RTE_SCHED_TYPE_PARALLEL,
 531                                                  queue, 0);
 532                 rte_event_enqueue_burst(evdev, 0, &ev, 1);
 533         }
 534
 535         return consume_events(0, max_evts_roundoff, validate_queue_priority);
 536 }
 537
 538 static int
 539 worker_multi_port_fn(void *arg)
 540 {
 541         struct test_core_param *param = arg;
 542         uint32_t *total_events = param->total_events;
 543         uint8_t port = param->port;
 544         uint16_t valid_event;
 545         struct rte_event ev;
 546         int ret;
 547
 548         while (__atomic_load_n(total_events, __ATOMIC_RELAXED) > 0) {
 549                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0);
 550                 if (!valid_event)
 551                         continue;
 552
 553                 ret = validate_event(&ev);
 554                 RTE_TEST_ASSERT_SUCCESS(ret, "Failed to validate event");
 555                 rte_pktmbuf_free(ev.mbuf);
 556                 __atomic_sub_fetch(total_events, 1, __ATOMIC_RELAXED);
 557         }
 558
 559         return 0;
 560 }
 561
 562 static inline int
 563 wait_workers_to_join(const uint32_t *count)
 564 {
 565         uint64_t cycles, print_cycles;
 566
 567         cycles = rte_get_timer_cycles();
 568         print_cycles = cycles;
 569         while (__atomic_load_n(count, __ATOMIC_RELAXED)) {
 570                 uint64_t new_cycles = rte_get_timer_cycles();
 571
 572                 if (new_cycles - print_cycles > rte_get_timer_hz()) {
 573                         plt_info("Events %d",
 574                                  __atomic_load_n(count, __ATOMIC_RELAXED));
 575                         print_cycles = new_cycles;
 576                 }
 577                 if (new_cycles - cycles > rte_get_timer_hz() * 10000000000) {
 578                         plt_err("No schedules for seconds, deadlock (%d)",
 579                                 __atomic_load_n(count, __ATOMIC_RELAXED));
 580                         rte_event_dev_dump(evdev, stdout);
 581                         cycles = new_cycles;
 582                         return -1;
 583                 }
 584         }
 585         rte_eal_mp_wait_lcore();
 586
 587         return 0;
 588 }
 589
 590 static inline int
 591 launch_workers_and_wait(int (*main_thread)(void *),
 592                         int (*worker_thread)(void *), uint32_t total_events,
 593                         uint8_t nb_workers, uint8_t sched_type)
 594 {
 595         uint32_t atomic_total_events;
 596         struct test_core_param *param;
 597         uint64_t dequeue_tmo_ticks;
 598         uint8_t port = 0;
 599         int w_lcore;
 600         int ret;
 601
 602         if (!nb_workers)
 603                 return 0;
 604
 605         __atomic_store_n(&atomic_total_events, total_events, __ATOMIC_RELAXED);
 606         seqn_list_init();
 607
 608         param = malloc(sizeof(struct test_core_param) * nb_workers);
 609         if (!param)
 610                 return -1;
 611
 612         ret = rte_event_dequeue_timeout_ticks(
 613                 evdev, rte_rand() % 10000000 /* 10ms */, &dequeue_tmo_ticks);
 614         if (ret) {
 615                 free(param);
 616                 return -1;
 617         }
 618
 619         param[0].total_events = &atomic_total_events;
 620         param[0].sched_type = sched_type;
 621         param[0].port = 0;
 622         param[0].dequeue_tmo_ticks = dequeue_tmo_ticks;
 623         rte_wmb();
 624
 625         w_lcore = rte_get_next_lcore(
 626                 /* start core */ -1,
 627                 /* skip main */ 1,
 628                 /* wrap */ 0);
 629         rte_eal_remote_launch(main_thread, &param[0], w_lcore);
 630
 631         for (port = 1; port < nb_workers; port++) {
 632                 param[port].total_events = &atomic_total_events;
 633                 param[port].sched_type = sched_type;
 634                 param[port].port = port;
 635                 param[port].dequeue_tmo_ticks = dequeue_tmo_ticks;
 636                 rte_atomic_thread_fence(__ATOMIC_RELEASE);
 637                 w_lcore = rte_get_next_lcore(w_lcore, 1, 0);
 638                 rte_eal_remote_launch(worker_thread, &param[port], w_lcore);
 639         }
 640
 641         rte_atomic_thread_fence(__ATOMIC_RELEASE);
 642         ret = wait_workers_to_join(&atomic_total_events);
 643         free(param);
 644
 645         return ret;
 646 }
 647
 648 /*
 649  * Generate a prescribed number of events and spread them across available
 650  * queues. Dequeue the events through multiple ports and verify the enqueued
 651  * event attributes
 652  */
 653 static int
 654 test_multi_queue_enq_multi_port_deq(void)
 655 {
 656         const unsigned int total_events = MAX_EVENTS;
 657         uint32_t nr_ports;
 658         int ret;
 659
 660         ret = generate_random_events(total_events);
 661         if (ret)
 662                 return -1;
 663
 664         RTE_TEST_ASSERT_SUCCESS(
 665                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_PORT_COUNT,
 666                                        &nr_ports),
 667                 "Port count get failed");
 668         nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1);
 669
 670         if (!nr_ports) {
 671                 plt_err("Not enough ports=%d or workers=%d", nr_ports,
 672                         rte_lcore_count() - 1);
 673                 return 0;
 674         }
 675
 676         return launch_workers_and_wait(worker_multi_port_fn,
 677                                        worker_multi_port_fn, total_events,
 678                                        nr_ports, 0xff /* invalid */);
 679 }
 680
 681 static void
 682 flush(uint8_t dev_id, struct rte_event event, void *arg)
 683 {
 684         unsigned int *count = arg;
 685
 686         RTE_SET_USED(dev_id);
 687         if (event.event_type == RTE_EVENT_TYPE_CPU)
 688                 *count = *count + 1;
 689 }
 690
 691 static int
 692 test_dev_stop_flush(void)
 693 {
 694         unsigned int total_events = MAX_EVENTS, count = 0;
 695         int ret;
 696
 697         ret = generate_random_events(total_events);
 698         if (ret)
 699                 return -1;
 700
 701         ret = rte_event_dev_stop_flush_callback_register(evdev, flush, &count);
 702         if (ret)
 703                 return -2;
 704         rte_event_dev_stop(evdev);
 705         ret = rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL);
 706         if (ret)
 707                 return -3;
 708         RTE_TEST_ASSERT_EQUAL(total_events, count,
 709                               "count mismatch total_events=%d count=%d",
 710                               total_events, count);
 711
 712         return 0;
 713 }
 714
 715 static int
 716 validate_queue_to_port_single_link(uint32_t index, uint8_t port,
 717                                    struct rte_event *ev)
 718 {
 719         RTE_SET_USED(index);
 720         RTE_TEST_ASSERT_EQUAL(port, ev->queue_id,
 721                               "queue mismatch enq=%d deq =%d", port,
 722                               ev->queue_id);
 723
 724         return 0;
 725 }
 726
 727 /*
 728  * Link queue x to port x and check correctness of link by checking
 729  * queue_id == x on dequeue on the specific port x
 730  */
 731 static int
 732 test_queue_to_port_single_link(void)
 733 {
 734         int i, nr_links, ret;
 735         uint32_t queue_count;
 736         uint32_t port_count;
 737
 738         RTE_TEST_ASSERT_SUCCESS(
 739                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_PORT_COUNT,
 740                                        &port_count),
 741                 "Port count get failed");
 742
 743         /* Unlink all connections that created in eventdev_setup */
 744         for (i = 0; i < (int)port_count; i++) {
 745                 ret = rte_event_port_unlink(evdev, i, NULL, 0);
 746                 RTE_TEST_ASSERT(ret >= 0, "Failed to unlink all queues port=%d",
 747                                 i);
 748         }
 749
 750         RTE_TEST_ASSERT_SUCCESS(
 751                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 752                                        &queue_count),
 753                 "Queue count get failed");
 754
 755         nr_links = RTE_MIN(port_count, queue_count);
 756         const unsigned int total_events = MAX_EVENTS / nr_links;
 757
 758         /* Link queue x to port x and inject events to queue x through port x */
 759         for (i = 0; i < nr_links; i++) {
 760                 uint8_t queue = (uint8_t)i;
 761
 762                 ret = rte_event_port_link(evdev, i, &queue, NULL, 1);
 763                 RTE_TEST_ASSERT(ret == 1, "Failed to link queue to port %d", i);
 764
 765                 ret = inject_events(0x100 /*flow_id */,
 766                                     RTE_EVENT_TYPE_CPU /* event_type */,
 767                                     rte_rand() % 256 /* sub_event_type */,
 768                                     rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1),
 769                                     queue /* queue */, i /* port */,
 770                                     total_events /* events */);
 771                 if (ret)
 772                         return -1;
 773         }
 774
 775         /* Verify the events generated from correct queue */
 776         for (i = 0; i < nr_links; i++) {
 777                 ret = consume_events(i /* port */, total_events,
 778                                      validate_queue_to_port_single_link);
 779                 if (ret)
 780                         return -1;
 781         }
 782
 783         return 0;
 784 }
 785
 786 static int
 787 validate_queue_to_port_multi_link(uint32_t index, uint8_t port,
 788                                   struct rte_event *ev)
 789 {
 790         RTE_SET_USED(index);
 791         RTE_TEST_ASSERT_EQUAL(port, (ev->queue_id & 0x1),
 792                               "queue mismatch enq=%d deq =%d", port,
 793                               ev->queue_id);
 794
 795         return 0;
 796 }
 797
 798 /*
 799  * Link all even number of queues to port 0 and all odd number of queues to
 800  * port 1 and verify the link connection on dequeue
 801  */
 802 static int
 803 test_queue_to_port_multi_link(void)
 804 {
 805         int ret, port0_events = 0, port1_events = 0;
 806         uint32_t nr_queues = 0;
 807         uint32_t nr_ports = 0;
 808         uint8_t queue, port;
 809
 810         RTE_TEST_ASSERT_SUCCESS(
 811                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 812                                        &nr_queues),
 813                 "Queue count get failed");
 814         RTE_TEST_ASSERT_SUCCESS(
 815                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 816                                        &nr_queues),
 817                 "Queue count get failed");
 818         RTE_TEST_ASSERT_SUCCESS(
 819                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_PORT_COUNT,
 820                                        &nr_ports),
 821                 "Port count get failed");
 822
 823         if (nr_ports < 2) {
 824                 plt_err("Not enough ports to test ports=%d", nr_ports);
 825                 return 0;
 826         }
 827
 828         /* Unlink all connections that created in eventdev_setup */
 829         for (port = 0; port < nr_ports; port++) {
 830                 ret = rte_event_port_unlink(evdev, port, NULL, 0);
 831                 RTE_TEST_ASSERT(ret >= 0, "Failed to unlink all queues port=%d",
 832                                 port);
 833         }
 834
 835         unsigned int total_events = MAX_EVENTS / nr_queues;
 836         if (!total_events) {
 837                 nr_queues = MAX_EVENTS;
 838                 total_events = MAX_EVENTS / nr_queues;
 839         }
 840
 841         /* Link all even number of queues to port0 and odd numbers to port 1*/
 842         for (queue = 0; queue < nr_queues; queue++) {
 843                 port = queue & 0x1;
 844                 ret = rte_event_port_link(evdev, port, &queue, NULL, 1);
 845                 RTE_TEST_ASSERT(ret == 1, "Failed to link queue=%d to port=%d",
 846                                 queue, port);
 847
 848                 ret = inject_events(0x100 /*flow_id */,
 849                                     RTE_EVENT_TYPE_CPU /* event_type */,
 850                                     rte_rand() % 256 /* sub_event_type */,
 851                                     rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1),
 852                                     queue /* queue */, port /* port */,
 853                                     total_events /* events */);
 854                 if (ret)
 855                         return -1;
 856
 857                 if (port == 0)
 858                         port0_events += total_events;
 859                 else
 860                         port1_events += total_events;
 861         }
 862
 863         ret = consume_events(0 /* port */, port0_events,
 864                              validate_queue_to_port_multi_link);
 865         if (ret)
 866                 return -1;
 867         ret = consume_events(1 /* port */, port1_events,
 868                              validate_queue_to_port_multi_link);
 869         if (ret)
 870                 return -1;
 871
 872         return 0;
 873 }
 874
 875 static int
 876 worker_flow_based_pipeline(void *arg)
 877 {
 878         struct test_core_param *param = arg;
 879         uint64_t dequeue_tmo_ticks = param->dequeue_tmo_ticks;
 880         uint32_t *total_events = param->total_events;
 881         uint8_t new_sched_type = param->sched_type;
 882         uint8_t port = param->port;
 883         uint16_t valid_event;
 884         struct rte_event ev;
 885
 886         while (__atomic_load_n(total_events, __ATOMIC_RELAXED) > 0) {
 887                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1,
 888                                                       dequeue_tmo_ticks);
 889                 if (!valid_event)
 890                         continue;
 891
 892                 /* Events from stage 0 */
 893                 if (ev.sub_event_type == 0) {
 894                         /* Move to atomic flow to maintain the ordering */
 895                         ev.flow_id = 0x2;
 896                         ev.event_type = RTE_EVENT_TYPE_CPU;
 897                         ev.sub_event_type = 1; /* stage 1 */
 898                         ev.sched_type = new_sched_type;
 899                         ev.op = RTE_EVENT_OP_FORWARD;
 900                         rte_event_enqueue_burst(evdev, port, &ev, 1);
 901                 } else if (ev.sub_event_type == 1) { /* Events from stage 1*/
 902                         uint32_t seqn = *rte_event_pmd_selftest_seqn(ev.mbuf);
 903
 904                         if (seqn_list_update(seqn) == 0) {
 905                                 rte_pktmbuf_free(ev.mbuf);
 906                                 __atomic_sub_fetch(total_events, 1,
 907                                                    __ATOMIC_RELAXED);
 908                         } else {
 909                                 plt_err("Failed to update seqn_list");
 910                                 return -1;
 911                         }
 912                 } else {
 913                         plt_err("Invalid ev.sub_event_type = %d",
 914                                 ev.sub_event_type);
 915                         return -1;
 916                 }
 917         }
 918         return 0;
 919 }
 920
 921 static int
 922 test_multiport_flow_sched_type_test(uint8_t in_sched_type,
 923                                     uint8_t out_sched_type)
 924 {
 925         const unsigned int total_events = MAX_EVENTS;
 926         uint32_t nr_ports;
 927         int ret;
 928
 929         RTE_TEST_ASSERT_SUCCESS(
 930                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_PORT_COUNT,
 931                                        &nr_ports),
 932                 "Port count get failed");
 933         nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1);
 934
 935         if (!nr_ports) {
 936                 plt_err("Not enough ports=%d or workers=%d", nr_ports,
 937                         rte_lcore_count() - 1);
 938                 return 0;
 939         }
 940
 941         /* Injects events with a 0 sequence number to total_events */
 942         ret = inject_events(
 943                 0x1 /*flow_id */, RTE_EVENT_TYPE_CPU /* event_type */,
 944                 0 /* sub_event_type (stage 0) */, in_sched_type, 0 /* queue */,
 945                 0 /* port */, total_events /* events */);
 946         if (ret)
 947                 return -1;
 948
 949         rte_mb();
 950         ret = launch_workers_and_wait(worker_flow_based_pipeline,
 951                                       worker_flow_based_pipeline, total_events,
 952                                       nr_ports, out_sched_type);
 953         if (ret)
 954                 return -1;
 955
 956         if (in_sched_type != RTE_SCHED_TYPE_PARALLEL &&
 957             out_sched_type == RTE_SCHED_TYPE_ATOMIC) {
 958                 /* Check the events order maintained or not */
 959                 return seqn_list_check(total_events);
 960         }
 961
 962         return 0;
 963 }
 964
 965 /* Multi port ordered to atomic transaction */
 966 static int
 967 test_multi_port_flow_ordered_to_atomic(void)
 968 {
 969         /* Ingress event order test */
 970         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ORDERED,
 971                                                    RTE_SCHED_TYPE_ATOMIC);
 972 }
 973
 974 static int
 975 test_multi_port_flow_ordered_to_ordered(void)
 976 {
 977         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ORDERED,
 978                                                    RTE_SCHED_TYPE_ORDERED);
 979 }
 980
 981 static int
 982 test_multi_port_flow_ordered_to_parallel(void)
 983 {
 984         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ORDERED,
 985                                                    RTE_SCHED_TYPE_PARALLEL);
 986 }
 987
 988 static int
 989 test_multi_port_flow_atomic_to_atomic(void)
 990 {
 991         /* Ingress event order test */
 992         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ATOMIC,
 993                                                    RTE_SCHED_TYPE_ATOMIC);
 994 }
 995
 996 static int
 997 test_multi_port_flow_atomic_to_ordered(void)
 998 {
 999         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ATOMIC,
1000                                                    RTE_SCHED_TYPE_ORDERED);
1001 }
1002
1003 static int
1004 test_multi_port_flow_atomic_to_parallel(void)
1005 {
1006         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ATOMIC,
1007                                                    RTE_SCHED_TYPE_PARALLEL);
1008 }
1009
1010 static int
1011 test_multi_port_flow_parallel_to_atomic(void)
1012 {
1013         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_PARALLEL,
1014                                                    RTE_SCHED_TYPE_ATOMIC);
1015 }
1016
1017 static int
1018 test_multi_port_flow_parallel_to_ordered(void)
1019 {
1020         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_PARALLEL,
1021                                                    RTE_SCHED_TYPE_ORDERED);
1022 }
1023
1024 static int
1025 test_multi_port_flow_parallel_to_parallel(void)
1026 {
1027         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_PARALLEL,
1028                                                    RTE_SCHED_TYPE_PARALLEL);
1029 }
1030
1031 static int
1032 worker_group_based_pipeline(void *arg)
1033 {
1034         struct test_core_param *param = arg;
1035         uint64_t dequeue_tmo_ticks = param->dequeue_tmo_ticks;
1036         uint32_t *total_events = param->total_events;
1037         uint8_t new_sched_type = param->sched_type;
1038         uint8_t port = param->port;
1039         uint16_t valid_event;
1040         struct rte_event ev;
1041
1042         while (__atomic_load_n(total_events, __ATOMIC_RELAXED) > 0) {
1043                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1,
1044                                                       dequeue_tmo_ticks);
1045                 if (!valid_event)
1046                         continue;
1047
1048                 /* Events from stage 0(group 0) */
1049                 if (ev.queue_id == 0) {
1050                         /* Move to atomic flow to maintain the ordering */
1051                         ev.flow_id = 0x2;
1052                         ev.event_type = RTE_EVENT_TYPE_CPU;
1053                         ev.sched_type = new_sched_type;
1054                         ev.queue_id = 1; /* Stage 1*/
1055                         ev.op = RTE_EVENT_OP_FORWARD;
1056                         rte_event_enqueue_burst(evdev, port, &ev, 1);
1057                 } else if (ev.queue_id == 1) { /* Events from stage 1(group 1)*/
1058                         uint32_t seqn = *rte_event_pmd_selftest_seqn(ev.mbuf);
1059
1060                         if (seqn_list_update(seqn) == 0) {
1061                                 rte_pktmbuf_free(ev.mbuf);
1062                                 __atomic_sub_fetch(total_events, 1,
1063                                                    __ATOMIC_RELAXED);
1064                         } else {
1065                                 plt_err("Failed to update seqn_list");
1066                                 return -1;
1067                         }
1068                 } else {
1069                         plt_err("Invalid ev.queue_id = %d", ev.queue_id);
1070                         return -1;
1071                 }
1072         }
1073
1074         return 0;
1075 }
1076
1077 static int
1078 test_multiport_queue_sched_type_test(uint8_t in_sched_type,
1079                                      uint8_t out_sched_type)
1080 {
1081         const unsigned int total_events = MAX_EVENTS;
1082         uint32_t queue_count;
1083         uint32_t nr_ports;
1084         int ret;
1085
1086         RTE_TEST_ASSERT_SUCCESS(
1087                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_PORT_COUNT,
1088                                        &nr_ports),
1089                 "Port count get failed");
1090
1091         nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1);
1092
1093         RTE_TEST_ASSERT_SUCCESS(
1094                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
1095                                        &queue_count),
1096                 "Queue count get failed");
1097         if (queue_count < 2 || !nr_ports) {
1098                 plt_err("Not enough queues=%d ports=%d or workers=%d",
1099                         queue_count, nr_ports, rte_lcore_count() - 1);
1100                 return 0;
1101         }
1102
1103         /* Injects events with a 0 sequence number to total_events */
1104         ret = inject_events(
1105                 0x1 /*flow_id */, RTE_EVENT_TYPE_CPU /* event_type */,
1106                 0 /* sub_event_type (stage 0) */, in_sched_type, 0 /* queue */,
1107                 0 /* port */, total_events /* events */);
1108         if (ret)
1109                 return -1;
1110
1111         ret = launch_workers_and_wait(worker_group_based_pipeline,
1112                                       worker_group_based_pipeline, total_events,
1113                                       nr_ports, out_sched_type);
1114         if (ret)
1115                 return -1;
1116
1117         if (in_sched_type != RTE_SCHED_TYPE_PARALLEL &&
1118             out_sched_type == RTE_SCHED_TYPE_ATOMIC) {
1119                 /* Check the events order maintained or not */
1120                 return seqn_list_check(total_events);
1121         }
1122
1123         return 0;
1124 }
1125
1126 static int
1127 test_multi_port_queue_ordered_to_atomic(void)
1128 {
1129         /* Ingress event order test */
1130         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ORDERED,
1131                                                     RTE_SCHED_TYPE_ATOMIC);
1132 }
1133
1134 static int
1135 test_multi_port_queue_ordered_to_ordered(void)
1136 {
1137         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ORDERED,
1138                                                     RTE_SCHED_TYPE_ORDERED);
1139 }
1140
1141 static int
1142 test_multi_port_queue_ordered_to_parallel(void)
1143 {
1144         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ORDERED,
1145                                                     RTE_SCHED_TYPE_PARALLEL);
1146 }
1147
1148 static int
1149 test_multi_port_queue_atomic_to_atomic(void)
1150 {
1151         /* Ingress event order test */
1152         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ATOMIC,
1153                                                     RTE_SCHED_TYPE_ATOMIC);
1154 }
1155
1156 static int
1157 test_multi_port_queue_atomic_to_ordered(void)
1158 {
1159         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ATOMIC,
1160                                                     RTE_SCHED_TYPE_ORDERED);
1161 }
1162
1163 static int
1164 test_multi_port_queue_atomic_to_parallel(void)
1165 {
1166         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ATOMIC,
1167                                                     RTE_SCHED_TYPE_PARALLEL);
1168 }
1169
1170 static int
1171 test_multi_port_queue_parallel_to_atomic(void)
1172 {
1173         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_PARALLEL,
1174                                                     RTE_SCHED_TYPE_ATOMIC);
1175 }
1176
1177 static int
1178 test_multi_port_queue_parallel_to_ordered(void)
1179 {
1180         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_PARALLEL,
1181                                                     RTE_SCHED_TYPE_ORDERED);
1182 }
1183
1184 static int
1185 test_multi_port_queue_parallel_to_parallel(void)
1186 {
1187         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_PARALLEL,
1188                                                     RTE_SCHED_TYPE_PARALLEL);
1189 }
1190
1191 static int
1192 worker_flow_based_pipeline_max_stages_rand_sched_type(void *arg)
1193 {
1194         struct test_core_param *param = arg;
1195         uint32_t *total_events = param->total_events;
1196         uint8_t port = param->port;
1197         uint16_t valid_event;
1198         struct rte_event ev;
1199
1200         while (__atomic_load_n(total_events, __ATOMIC_RELAXED) > 0) {
1201                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0);
1202                 if (!valid_event)
1203                         continue;
1204
1205                 if (ev.sub_event_type == MAX_STAGES) { /* last stage */
1206                         rte_pktmbuf_free(ev.mbuf);
1207                         __atomic_sub_fetch(total_events, 1, __ATOMIC_RELAXED);
1208                 } else {
1209                         ev.event_type = RTE_EVENT_TYPE_CPU;
1210                         ev.sub_event_type++;
1211                         ev.sched_type =
1212                                 rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1);
1213                         ev.op = RTE_EVENT_OP_FORWARD;
1214                         rte_event_enqueue_burst(evdev, port, &ev, 1);
1215                 }
1216         }
1217
1218         return 0;
1219 }
1220
1221 static int
1222 launch_multi_port_max_stages_random_sched_type(int (*fn)(void *))
1223 {
1224         uint32_t nr_ports;
1225         int ret;
1226
1227         RTE_TEST_ASSERT_SUCCESS(
1228                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_PORT_COUNT,
1229                                        &nr_ports),
1230                 "Port count get failed");
1231         nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1);
1232
1233         if (!nr_ports) {
1234                 plt_err("Not enough ports=%d or workers=%d", nr_ports,
1235                         rte_lcore_count() - 1);
1236                 return 0;
1237         }
1238
1239         /* Injects events with a 0 sequence number to total_events */
1240         ret = inject_events(
1241                 0x1 /*flow_id */, RTE_EVENT_TYPE_CPU /* event_type */,
1242                 0 /* sub_event_type (stage 0) */,
1243                 rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1) /* sched_type */,
1244                 0 /* queue */, 0 /* port */, MAX_EVENTS /* events */);
1245         if (ret)
1246                 return -1;
1247
1248         return launch_workers_and_wait(fn, fn, MAX_EVENTS, nr_ports,
1249                                        0xff /* invalid */);
1250 }
1251
1252 /* Flow based pipeline with maximum stages with random sched type */
1253 static int
1254 test_multi_port_flow_max_stages_random_sched_type(void)
1255 {
1256         return launch_multi_port_max_stages_random_sched_type(
1257                 worker_flow_based_pipeline_max_stages_rand_sched_type);
1258 }
1259
1260 static int
1261 worker_queue_based_pipeline_max_stages_rand_sched_type(void *arg)
1262 {
1263         struct test_core_param *param = arg;
1264         uint8_t port = param->port;
1265         uint32_t queue_count;
1266         uint16_t valid_event;
1267         struct rte_event ev;
1268
1269         RTE_TEST_ASSERT_SUCCESS(
1270                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
1271                                        &queue_count),
1272                 "Queue count get failed");
1273         uint8_t nr_queues = queue_count;
1274         uint32_t *total_events = param->total_events;
1275
1276         while (__atomic_load_n(total_events, __ATOMIC_RELAXED) > 0) {
1277                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0);
1278                 if (!valid_event)
1279                         continue;
1280
1281                 if (ev.queue_id == nr_queues - 1) { /* last stage */
1282                         rte_pktmbuf_free(ev.mbuf);
1283                         __atomic_sub_fetch(total_events, 1, __ATOMIC_RELAXED);
1284                 } else {
1285                         ev.event_type = RTE_EVENT_TYPE_CPU;
1286                         ev.queue_id++;
1287                         ev.sched_type =
1288                                 rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1);
1289                         ev.op = RTE_EVENT_OP_FORWARD;
1290                         rte_event_enqueue_burst(evdev, port, &ev, 1);
1291                 }
1292         }
1293
1294         return 0;
1295 }
1296
1297 /* Queue based pipeline with maximum stages with random sched type */
1298 static int
1299 test_multi_port_queue_max_stages_random_sched_type(void)
1300 {
1301         return launch_multi_port_max_stages_random_sched_type(
1302                 worker_queue_based_pipeline_max_stages_rand_sched_type);
1303 }
1304
1305 static int
1306 worker_mixed_pipeline_max_stages_rand_sched_type(void *arg)
1307 {
1308         struct test_core_param *param = arg;
1309         uint8_t port = param->port;
1310         uint32_t queue_count;
1311         uint16_t valid_event;
1312         struct rte_event ev;
1313
1314         RTE_TEST_ASSERT_SUCCESS(
1315                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
1316                                        &queue_count),
1317                 "Queue count get failed");
1318         uint8_t nr_queues = queue_count;
1319         uint32_t *total_events = param->total_events;
1320
1321         while (__atomic_load_n(total_events, __ATOMIC_RELAXED) > 0) {
1322                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0);
1323                 if (!valid_event)
1324                         continue;
1325
1326                 if (ev.queue_id == nr_queues - 1) { /* Last stage */
1327                         rte_pktmbuf_free(ev.mbuf);
1328                         __atomic_sub_fetch(total_events, 1, __ATOMIC_RELAXED);
1329                 } else {
1330                         ev.event_type = RTE_EVENT_TYPE_CPU;
1331                         ev.queue_id++;
1332                         ev.sub_event_type = rte_rand() % 256;
1333                         ev.sched_type =
1334                                 rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1);
1335                         ev.op = RTE_EVENT_OP_FORWARD;
1336                         rte_event_enqueue_burst(evdev, port, &ev, 1);
1337                 }
1338         }
1339
1340         return 0;
1341 }
1342
1343 /* Queue and flow based pipeline with maximum stages with random sched type */
1344 static int
1345 test_multi_port_mixed_max_stages_random_sched_type(void)
1346 {
1347         return launch_multi_port_max_stages_random_sched_type(
1348                 worker_mixed_pipeline_max_stages_rand_sched_type);
1349 }
1350
1351 static int
1352 worker_ordered_flow_producer(void *arg)
1353 {
1354         struct test_core_param *param = arg;
1355         uint8_t port = param->port;
1356         struct rte_mbuf *m;
1357         int counter = 0;
1358
1359         while (counter < NUM_PACKETS) {
1360                 m = rte_pktmbuf_alloc(eventdev_test_mempool);
1361                 if (m == NULL)
1362                         continue;
1363
1364                 *rte_event_pmd_selftest_seqn(m) = counter++;
1365
1366                 struct rte_event ev = {.event = 0, .u64 = 0};
1367
1368                 ev.flow_id = 0x1; /* Generate a fat flow */
1369                 ev.sub_event_type = 0;
1370                 /* Inject the new event */
1371                 ev.op = RTE_EVENT_OP_NEW;
1372                 ev.event_type = RTE_EVENT_TYPE_CPU;
1373                 ev.sched_type = RTE_SCHED_TYPE_ORDERED;
1374                 ev.queue_id = 0;
1375                 ev.mbuf = m;
1376                 rte_event_enqueue_burst(evdev, port, &ev, 1);
1377         }
1378
1379         return 0;
1380 }
1381
1382 static inline int
1383 test_producer_consumer_ingress_order_test(int (*fn)(void *))
1384 {
1385         uint32_t nr_ports;
1386
1387         RTE_TEST_ASSERT_SUCCESS(
1388                 rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_PORT_COUNT,
1389                                        &nr_ports),
1390                 "Port count get failed");
1391         nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1);
1392
1393         if (rte_lcore_count() < 3 || nr_ports < 2) {
1394                 plt_err("### Not enough cores for test.");
1395                 return 0;
1396         }
1397
1398         launch_workers_and_wait(worker_ordered_flow_producer, fn, NUM_PACKETS,
1399                                 nr_ports, RTE_SCHED_TYPE_ATOMIC);
1400         /* Check the events order maintained or not */
1401         return seqn_list_check(NUM_PACKETS);
1402 }
1403
1404 /* Flow based producer consumer ingress order test */
1405 static int
1406 test_flow_producer_consumer_ingress_order_test(void)
1407 {
1408         return test_producer_consumer_ingress_order_test(
1409                 worker_flow_based_pipeline);
1410 }
1411
1412 /* Queue based producer consumer ingress order test */
1413 static int
1414 test_queue_producer_consumer_ingress_order_test(void)
1415 {
1416         return test_producer_consumer_ingress_order_test(
1417                 worker_group_based_pipeline);
1418 }
1419
1420 static void
1421 cnxk_test_run(int (*setup)(void), void (*tdown)(void), int (*test)(void),
1422               const char *name)
1423 {
1424         if (setup() < 0) {
1425                 printf("Error setting up test %s", name);
1426                 unsupported++;
1427         } else {
1428                 if (test() < 0) {
1429                         failed++;
1430                         printf("+ TestCase [%2d] : %s failed\n", total, name);
1431                 } else {
1432                         passed++;
1433                         printf("+ TestCase [%2d] : %s succeeded\n", total,
1434                                name);
1435                 }
1436         }
1437
1438         total++;
1439         tdown();
1440 }
1441
1442 static int
1443 cnxk_sso_testsuite_run(const char *dev_name)
1444 {
1445         int rc;
1446
1447         testsuite_setup(dev_name);
1448
1449         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1450                       test_simple_enqdeq_ordered);
1451         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1452                       test_simple_enqdeq_atomic);
1453         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1454                       test_simple_enqdeq_parallel);
1455         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1456                       test_multi_queue_enq_single_port_deq);
1457         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown, test_dev_stop_flush);
1458         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1459                       test_multi_queue_enq_multi_port_deq);
1460         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1461                       test_queue_to_port_single_link);
1462         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1463                       test_queue_to_port_multi_link);
1464         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1465                       test_multi_port_flow_ordered_to_atomic);
1466         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1467                       test_multi_port_flow_ordered_to_ordered);
1468         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1469                       test_multi_port_flow_ordered_to_parallel);
1470         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1471                       test_multi_port_flow_atomic_to_atomic);
1472         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1473                       test_multi_port_flow_atomic_to_ordered);
1474         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1475                       test_multi_port_flow_atomic_to_parallel);
1476         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1477                       test_multi_port_flow_parallel_to_atomic);
1478         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1479                       test_multi_port_flow_parallel_to_ordered);
1480         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1481                       test_multi_port_flow_parallel_to_parallel);
1482         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1483                       test_multi_port_queue_ordered_to_atomic);
1484         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1485                       test_multi_port_queue_ordered_to_ordered);
1486         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1487                       test_multi_port_queue_ordered_to_parallel);
1488         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1489                       test_multi_port_queue_atomic_to_atomic);
1490         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1491                       test_multi_port_queue_atomic_to_ordered);
1492         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1493                       test_multi_port_queue_atomic_to_parallel);
1494         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1495                       test_multi_port_queue_parallel_to_atomic);
1496         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1497                       test_multi_port_queue_parallel_to_ordered);
1498         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1499                       test_multi_port_queue_parallel_to_parallel);
1500         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1501                       test_multi_port_flow_max_stages_random_sched_type);
1502         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1503                       test_multi_port_queue_max_stages_random_sched_type);
1504         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1505                       test_multi_port_mixed_max_stages_random_sched_type);
1506         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1507                       test_flow_producer_consumer_ingress_order_test);
1508         CNXK_TEST_RUN(eventdev_setup, eventdev_teardown,
1509                       test_queue_producer_consumer_ingress_order_test);
1510         CNXK_TEST_RUN(eventdev_setup_priority, eventdev_teardown,
1511                       test_multi_queue_priority);
1512         CNXK_TEST_RUN(eventdev_setup_dequeue_timeout, eventdev_teardown,
1513                       test_multi_port_flow_ordered_to_atomic);
1514         CNXK_TEST_RUN(eventdev_setup_dequeue_timeout, eventdev_teardown,
1515                       test_multi_port_queue_ordered_to_atomic);
1516         printf("Total tests   : %d\n", total);
1517         printf("Passed        : %d\n", passed);
1518         printf("Failed        : %d\n", failed);
1519         printf("Not supported : %d\n", unsupported);
1520
1521         rc = failed;
1522         testsuite_teardown();
1523
1524         return rc;
1525 }
1526
1527 int
1528 cnxk_sso_selftest(const char *dev_name)
1529 {
1530         const struct rte_memzone *mz;
1531         struct cnxk_sso_evdev *dev;
1532         int rc = -1;
1533
1534         mz = rte_memzone_lookup(CNXK_SSO_MZ_NAME);
1535         if (mz == NULL)
1536                 return rc;
1537
1538         dev = (void *)*((uint64_t *)mz->addr);
1539         if (roc_model_runtime_is_cn9k()) {
1540                 /* Verify single ws mode. */
1541                 printf("Verifying CN9K Single workslot mode\n");
1542                 dev->dual_ws = 0;
1543                 cn9k_sso_set_rsrc(dev);
1544                 if (cnxk_sso_testsuite_run(dev_name))
1545                         return rc;
1546                 /* Verift dual ws mode. */
1547                 printf("Verifying CN9K Dual workslot mode\n");
1548                 dev->dual_ws = 1;
1549                 cn9k_sso_set_rsrc(dev);
1550                 if (cnxk_sso_testsuite_run(dev_name))
1551                         return rc;
1552         }
1553
1554         if (roc_model_runtime_is_cn10k()) {
1555                 printf("Verifying CN10K workslot getwork mode none\n");
1556                 dev->gw_mode = CN10K_GW_MODE_NONE;
1557                 if (cnxk_sso_testsuite_run(dev_name))
1558                         return rc;
1559                 printf("Verifying CN10K workslot getwork mode prefetch\n");
1560                 dev->gw_mode = CN10K_GW_MODE_PREF;
1561                 if (cnxk_sso_testsuite_run(dev_name))
1562                         return rc;
1563                 printf("Verifying CN10K workslot getwork mode smart prefetch\n");
1564                 dev->gw_mode = CN10K_GW_MODE_PREF_WFE;
1565                 if (cnxk_sso_testsuite_run(dev_name))
1566                         return rc;
1567         }
1568
1569         return 0;
1570 }