app/test-bbdev/test_bbdev_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7 #include <math.h>
   8
   9 #include <rte_eal.h>
  10 #include <rte_common.h>
  11 #include <rte_dev.h>
  12 #include <rte_launch.h>
  13 #include <rte_bbdev.h>
  14 #include <rte_cycles.h>
  15 #include <rte_lcore.h>
  16 #include <rte_malloc.h>
  17 #include <rte_random.h>
  18 #include <rte_hexdump.h>
  19
  20 #include "main.h"
  21 #include "test_bbdev_vector.h"
  22
  23 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
  24
  25 #define MAX_QUEUES RTE_MAX_LCORE
  26
  27 #define OPS_CACHE_SIZE 256U
  28 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
  29
  30 #define SYNC_WAIT 0
  31 #define SYNC_START 1
  32
  33 #define INVALID_QUEUE_ID -1
  34
  35 static struct test_bbdev_vector test_vector;
  36
  37 /* Switch between PMD and Interrupt for throughput TC */
  38 static bool intr_enabled;
  39
  40 /* Represents tested active devices */
  41 static struct active_device {
  42         const char *driver_name;
  43         uint8_t dev_id;
  44         uint16_t supported_ops;
  45         uint16_t queue_ids[MAX_QUEUES];
  46         uint16_t nb_queues;
  47         struct rte_mempool *ops_mempool;
  48         struct rte_mempool *in_mbuf_pool;
  49         struct rte_mempool *hard_out_mbuf_pool;
  50         struct rte_mempool *soft_out_mbuf_pool;
  51 } active_devs[RTE_BBDEV_MAX_DEVS];
  52
  53 static uint8_t nb_active_devs;
  54
  55 /* Data buffers used by BBDEV ops */
  56 struct test_buffers {
  57         struct rte_bbdev_op_data *inputs;
  58         struct rte_bbdev_op_data *hard_outputs;
  59         struct rte_bbdev_op_data *soft_outputs;
  60 };
  61
  62 /* Operation parameters specific for given test case */
  63 struct test_op_params {
  64         struct rte_mempool *mp;
  65         struct rte_bbdev_dec_op *ref_dec_op;
  66         struct rte_bbdev_enc_op *ref_enc_op;
  67         uint16_t burst_sz;
  68         uint16_t num_to_process;
  69         uint16_t num_lcores;
  70         int vector_mask;
  71         rte_atomic16_t sync;
  72         struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
  73 };
  74
  75 /* Contains per lcore params */
  76 struct thread_params {
  77         uint8_t dev_id;
  78         uint16_t queue_id;
  79         uint64_t start_time;
  80         double mops;
  81         double mbps;
  82         rte_atomic16_t nb_dequeued;
  83         rte_atomic16_t processing_status;
  84         struct test_op_params *op_params;
  85 };
  86
  87 #ifdef RTE_BBDEV_OFFLOAD_COST
  88 /* Stores time statistics */
  89 struct test_time_stats {
  90         /* Stores software enqueue total working time */
  91         uint64_t enq_sw_tot_time;
  92         /* Stores minimum value of software enqueue working time */
  93         uint64_t enq_sw_min_time;
  94         /* Stores maximum value of software enqueue working time */
  95         uint64_t enq_sw_max_time;
  96         /* Stores turbo enqueue total working time */
  97         uint64_t enq_tur_tot_time;
  98         /* Stores minimum value of turbo enqueue working time */
  99         uint64_t enq_tur_min_time;
 100         /* Stores maximum value of turbo enqueue working time */
 101         uint64_t enq_tur_max_time;
 102         /* Stores dequeue total working time */
 103         uint64_t deq_tot_time;
 104         /* Stores minimum value of dequeue working time */
 105         uint64_t deq_min_time;
 106         /* Stores maximum value of dequeue working time */
 107         uint64_t deq_max_time;
 108 };
 109 #endif
 110
 111 typedef int (test_case_function)(struct active_device *ad,
 112                 struct test_op_params *op_params);
 113
 114 static inline void
 115 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 116 {
 117         ad->supported_ops |= (1 << op_type);
 118 }
 119
 120 static inline bool
 121 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 122 {
 123         return ad->supported_ops & (1 << op_type);
 124 }
 125
 126 static inline bool
 127 flags_match(uint32_t flags_req, uint32_t flags_present)
 128 {
 129         return (flags_req & flags_present) == flags_req;
 130 }
 131
 132 static void
 133 clear_soft_out_cap(uint32_t *op_flags)
 134 {
 135         *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
 136         *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
 137         *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
 138 }
 139
 140 static int
 141 check_dev_cap(const struct rte_bbdev_info *dev_info)
 142 {
 143         unsigned int i;
 144         unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs;
 145         const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
 146
 147         nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
 148         nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
 149         nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
 150
 151         for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
 152                 if (op_cap->type != test_vector.op_type)
 153                         continue;
 154
 155                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
 156                         const struct rte_bbdev_op_cap_turbo_dec *cap =
 157                                         &op_cap->cap.turbo_dec;
 158                         /* Ignore lack of soft output capability, just skip
 159                          * checking if soft output is valid.
 160                          */
 161                         if ((test_vector.turbo_dec.op_flags &
 162                                         RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
 163                                         !(cap->capability_flags &
 164                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 165                                 printf(
 166                                         "WARNING: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
 167                                         dev_info->dev_name);
 168                                 clear_soft_out_cap(
 169                                         &test_vector.turbo_dec.op_flags);
 170                         }
 171
 172                         if (!flags_match(test_vector.turbo_dec.op_flags,
 173                                         cap->capability_flags))
 174                                 return TEST_FAILED;
 175                         if (nb_inputs > cap->num_buffers_src) {
 176                                 printf("Too many inputs defined: %u, max: %u\n",
 177                                         nb_inputs, cap->num_buffers_src);
 178                                 return TEST_FAILED;
 179                         }
 180                         if (nb_soft_outputs > cap->num_buffers_soft_out &&
 181                                         (test_vector.turbo_dec.op_flags &
 182                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 183                                 printf(
 184                                         "Too many soft outputs defined: %u, max: %u\n",
 185                                                 nb_soft_outputs,
 186                                                 cap->num_buffers_soft_out);
 187                                 return TEST_FAILED;
 188                         }
 189                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 190                                 printf(
 191                                         "Too many hard outputs defined: %u, max: %u\n",
 192                                                 nb_hard_outputs,
 193                                                 cap->num_buffers_hard_out);
 194                                 return TEST_FAILED;
 195                         }
 196                         if (intr_enabled && !(cap->capability_flags &
 197                                         RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
 198                                 printf(
 199                                         "Dequeue interrupts are not supported!\n");
 200                                 return TEST_FAILED;
 201                         }
 202
 203                         return TEST_SUCCESS;
 204                 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
 205                         const struct rte_bbdev_op_cap_turbo_enc *cap =
 206                                         &op_cap->cap.turbo_enc;
 207
 208                         if (!flags_match(test_vector.turbo_enc.op_flags,
 209                                         cap->capability_flags))
 210                                 return TEST_FAILED;
 211                         if (nb_inputs > cap->num_buffers_src) {
 212                                 printf("Too many inputs defined: %u, max: %u\n",
 213                                         nb_inputs, cap->num_buffers_src);
 214                                 return TEST_FAILED;
 215                         }
 216                         if (nb_hard_outputs > cap->num_buffers_dst) {
 217                                 printf(
 218                                         "Too many hard outputs defined: %u, max: %u\n",
 219                                         nb_hard_outputs, cap->num_buffers_src);
 220                                 return TEST_FAILED;
 221                         }
 222                         if (intr_enabled && !(cap->capability_flags &
 223                                         RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
 224                                 printf(
 225                                         "Dequeue interrupts are not supported!\n");
 226                                 return TEST_FAILED;
 227                         }
 228
 229                         return TEST_SUCCESS;
 230                 }
 231         }
 232
 233         if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
 234                 return TEST_SUCCESS; /* Special case for NULL device */
 235
 236         return TEST_FAILED;
 237 }
 238
 239 /* calculates optimal mempool size not smaller than the val */
 240 static unsigned int
 241 optimal_mempool_size(unsigned int val)
 242 {
 243         return rte_align32pow2(val + 1) - 1;
 244 }
 245
 246 /* allocates mbuf mempool for inputs and outputs */
 247 static struct rte_mempool *
 248 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
 249                 int socket_id, unsigned int mbuf_pool_size,
 250                 const char *op_type_str)
 251 {
 252         unsigned int i;
 253         uint32_t max_seg_sz = 0;
 254         char pool_name[RTE_MEMPOOL_NAMESIZE];
 255
 256         /* find max input segment size */
 257         for (i = 0; i < entries->nb_segments; ++i)
 258                 if (entries->segments[i].length > max_seg_sz)
 259                         max_seg_sz = entries->segments[i].length;
 260
 261         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 262                         dev_id);
 263         return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
 264                         RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM,
 265                         (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
 266 }
 267
 268 static int
 269 create_mempools(struct active_device *ad, int socket_id,
 270                 enum rte_bbdev_op_type op_type, uint16_t num_ops)
 271 {
 272         struct rte_mempool *mp;
 273         unsigned int ops_pool_size, mbuf_pool_size = 0;
 274         char pool_name[RTE_MEMPOOL_NAMESIZE];
 275         const char *op_type_str;
 276
 277         struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
 278         struct op_data_entries *hard_out =
 279                         &test_vector.entries[DATA_HARD_OUTPUT];
 280         struct op_data_entries *soft_out =
 281                         &test_vector.entries[DATA_SOFT_OUTPUT];
 282
 283         /* allocate ops mempool */
 284         ops_pool_size = optimal_mempool_size(RTE_MAX(
 285                         /* Ops used plus 1 reference op */
 286                         RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
 287                         /* Minimal cache size plus 1 reference op */
 288                         (unsigned int)(1.5 * rte_lcore_count() *
 289                                         OPS_CACHE_SIZE + 1)),
 290                         OPS_POOL_SIZE_MIN));
 291
 292         op_type_str = rte_bbdev_op_type_str(op_type);
 293         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
 294
 295         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 296                         ad->dev_id);
 297         mp = rte_bbdev_op_pool_create(pool_name, op_type,
 298                         ops_pool_size, OPS_CACHE_SIZE, socket_id);
 299         TEST_ASSERT_NOT_NULL(mp,
 300                         "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
 301                         ops_pool_size,
 302                         ad->dev_id,
 303                         socket_id);
 304         ad->ops_mempool = mp;
 305
 306         /* Inputs */
 307         mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments);
 308         mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in");
 309         TEST_ASSERT_NOT_NULL(mp,
 310                         "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
 311                         mbuf_pool_size,
 312                         ad->dev_id,
 313                         socket_id);
 314         ad->in_mbuf_pool = mp;
 315
 316         /* Hard outputs */
 317         mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 318                         hard_out->nb_segments);
 319         mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size,
 320                         "hard_out");
 321         TEST_ASSERT_NOT_NULL(mp,
 322                         "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
 323                         mbuf_pool_size,
 324                         ad->dev_id,
 325                         socket_id);
 326         ad->hard_out_mbuf_pool = mp;
 327
 328         if (soft_out->nb_segments == 0)
 329                 return TEST_SUCCESS;
 330
 331         /* Soft outputs */
 332         mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 333                         soft_out->nb_segments);
 334         mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, mbuf_pool_size,
 335                         "soft_out");
 336         TEST_ASSERT_NOT_NULL(mp,
 337                         "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
 338                         mbuf_pool_size,
 339                         ad->dev_id,
 340                         socket_id);
 341         ad->soft_out_mbuf_pool = mp;
 342
 343         return 0;
 344 }
 345
 346 static int
 347 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
 348                 struct test_bbdev_vector *vector)
 349 {
 350         int ret;
 351         unsigned int queue_id;
 352         struct rte_bbdev_queue_conf qconf;
 353         struct active_device *ad = &active_devs[nb_active_devs];
 354         unsigned int nb_queues;
 355         enum rte_bbdev_op_type op_type = vector->op_type;
 356
 357         nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
 358         /* setup device */
 359         ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
 360         if (ret < 0) {
 361                 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
 362                                 dev_id, nb_queues, info->socket_id, ret);
 363                 return TEST_FAILED;
 364         }
 365
 366         /* configure interrupts if needed */
 367         if (intr_enabled) {
 368                 ret = rte_bbdev_intr_enable(dev_id);
 369                 if (ret < 0) {
 370                         printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
 371                                         ret);
 372                         return TEST_FAILED;
 373                 }
 374         }
 375
 376         /* setup device queues */
 377         qconf.socket = info->socket_id;
 378         qconf.queue_size = info->drv.default_queue_conf.queue_size;
 379         qconf.priority = 0;
 380         qconf.deferred_start = 0;
 381         qconf.op_type = op_type;
 382
 383         for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
 384                 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
 385                 if (ret != 0) {
 386                         printf(
 387                                         "Allocated all queues (id=%u) at prio%u on dev%u\n",
 388                                         queue_id, qconf.priority, dev_id);
 389                         qconf.priority++;
 390                         ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
 391                                         &qconf);
 392                 }
 393                 if (ret != 0) {
 394                         printf("All queues on dev %u allocated: %u\n",
 395                                         dev_id, queue_id);
 396                         break;
 397                 }
 398                 ad->queue_ids[queue_id] = queue_id;
 399         }
 400         TEST_ASSERT(queue_id != 0,
 401                         "ERROR Failed to configure any queues on dev %u",
 402                         dev_id);
 403         ad->nb_queues = queue_id;
 404
 405         set_avail_op(ad, op_type);
 406
 407         return TEST_SUCCESS;
 408 }
 409
 410 static int
 411 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
 412                 struct test_bbdev_vector *vector)
 413 {
 414         int ret;
 415
 416         active_devs[nb_active_devs].driver_name = info->drv.driver_name;
 417         active_devs[nb_active_devs].dev_id = dev_id;
 418
 419         ret = add_bbdev_dev(dev_id, info, vector);
 420         if (ret == TEST_SUCCESS)
 421                 ++nb_active_devs;
 422         return ret;
 423 }
 424
 425 static uint8_t
 426 populate_active_devices(void)
 427 {
 428         int ret;
 429         uint8_t dev_id;
 430         uint8_t nb_devs_added = 0;
 431         struct rte_bbdev_info info;
 432
 433         RTE_BBDEV_FOREACH(dev_id) {
 434                 rte_bbdev_info_get(dev_id, &info);
 435
 436                 if (check_dev_cap(&info)) {
 437                         printf(
 438                                 "Device %d (%s) does not support specified capabilities\n",
 439                                         dev_id, info.dev_name);
 440                         continue;
 441                 }
 442
 443                 ret = add_active_device(dev_id, &info, &test_vector);
 444                 if (ret != 0) {
 445                         printf("Adding active bbdev %s skipped\n",
 446                                         info.dev_name);
 447                         continue;
 448                 }
 449                 nb_devs_added++;
 450         }
 451
 452         return nb_devs_added;
 453 }
 454
 455 static int
 456 read_test_vector(void)
 457 {
 458         int ret;
 459
 460         memset(&test_vector, 0, sizeof(test_vector));
 461         printf("Test vector file = %s\n", get_vector_filename());
 462         ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
 463         TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
 464                         get_vector_filename());
 465
 466         return TEST_SUCCESS;
 467 }
 468
 469 static int
 470 testsuite_setup(void)
 471 {
 472         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 473
 474         if (populate_active_devices() == 0) {
 475                 printf("No suitable devices found!\n");
 476                 return TEST_SKIPPED;
 477         }
 478
 479         return TEST_SUCCESS;
 480 }
 481
 482 static int
 483 interrupt_testsuite_setup(void)
 484 {
 485         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 486
 487         /* Enable interrupts */
 488         intr_enabled = true;
 489
 490         /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
 491         if (populate_active_devices() == 0 ||
 492                         test_vector.op_type == RTE_BBDEV_OP_NONE) {
 493                 intr_enabled = false;
 494                 printf("No suitable devices found!\n");
 495                 return TEST_SKIPPED;
 496         }
 497
 498         return TEST_SUCCESS;
 499 }
 500
 501 static void
 502 testsuite_teardown(void)
 503 {
 504         uint8_t dev_id;
 505
 506         /* Unconfigure devices */
 507         RTE_BBDEV_FOREACH(dev_id)
 508                 rte_bbdev_close(dev_id);
 509
 510         /* Clear active devices structs. */
 511         memset(active_devs, 0, sizeof(active_devs));
 512         nb_active_devs = 0;
 513 }
 514
 515 static int
 516 ut_setup(void)
 517 {
 518         uint8_t i, dev_id;
 519
 520         for (i = 0; i < nb_active_devs; i++) {
 521                 dev_id = active_devs[i].dev_id;
 522                 /* reset bbdev stats */
 523                 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
 524                                 "Failed to reset stats of bbdev %u", dev_id);
 525                 /* start the device */
 526                 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
 527                                 "Failed to start bbdev %u", dev_id);
 528         }
 529
 530         return TEST_SUCCESS;
 531 }
 532
 533 static void
 534 ut_teardown(void)
 535 {
 536         uint8_t i, dev_id;
 537         struct rte_bbdev_stats stats;
 538
 539         for (i = 0; i < nb_active_devs; i++) {
 540                 dev_id = active_devs[i].dev_id;
 541                 /* read stats and print */
 542                 rte_bbdev_stats_get(dev_id, &stats);
 543                 /* Stop the device */
 544                 rte_bbdev_stop(dev_id);
 545         }
 546 }
 547
 548 static int
 549 init_op_data_objs(struct rte_bbdev_op_data *bufs,
 550                 struct op_data_entries *ref_entries,
 551                 struct rte_mempool *mbuf_pool, const uint16_t n,
 552                 enum op_data_type op_type, uint16_t min_alignment)
 553 {
 554         int ret;
 555         unsigned int i, j;
 556
 557         for (i = 0; i < n; ++i) {
 558                 char *data;
 559                 struct op_data_buf *seg = &ref_entries->segments[0];
 560                 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
 561                 TEST_ASSERT_NOT_NULL(m_head,
 562                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 563                                 op_type, n * ref_entries->nb_segments,
 564                                 mbuf_pool->size);
 565
 566                 bufs[i].data = m_head;
 567                 bufs[i].offset = 0;
 568                 bufs[i].length = 0;
 569
 570                 if (op_type == DATA_INPUT) {
 571                         data = rte_pktmbuf_append(m_head, seg->length);
 572                         TEST_ASSERT_NOT_NULL(data,
 573                                         "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 574                                         seg->length, op_type);
 575
 576                         TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment),
 577                                         "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 578                                         data, min_alignment);
 579                         rte_memcpy(data, seg->addr, seg->length);
 580                         bufs[i].length += seg->length;
 581
 582
 583                         for (j = 1; j < ref_entries->nb_segments; ++j) {
 584                                 struct rte_mbuf *m_tail =
 585                                                 rte_pktmbuf_alloc(mbuf_pool);
 586                                 TEST_ASSERT_NOT_NULL(m_tail,
 587                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 588                                                 op_type,
 589                                                 n * ref_entries->nb_segments,
 590                                                 mbuf_pool->size);
 591                                 seg += 1;
 592
 593                                 data = rte_pktmbuf_append(m_tail, seg->length);
 594                                 TEST_ASSERT_NOT_NULL(data,
 595                                                 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 596                                                 seg->length, op_type);
 597
 598                                 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
 599                                                 min_alignment),
 600                                                 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 601                                                 data, min_alignment);
 602                                 rte_memcpy(data, seg->addr, seg->length);
 603                                 bufs[i].length += seg->length;
 604
 605                                 ret = rte_pktmbuf_chain(m_head, m_tail);
 606                                 TEST_ASSERT_SUCCESS(ret,
 607                                                 "Couldn't chain mbufs from %d data type mbuf pool",
 608                                                 op_type);
 609                         }
 610                 }
 611         }
 612
 613         return 0;
 614 }
 615
 616 static int
 617 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
 618                 const int socket)
 619 {
 620         int i;
 621
 622         *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
 623         if (*buffers == NULL) {
 624                 printf("WARNING: Failed to allocate op_data on socket %d\n",
 625                                 socket);
 626                 /* try to allocate memory on other detected sockets */
 627                 for (i = 0; i < socket; i++) {
 628                         *buffers = rte_zmalloc_socket(NULL, len, 0, i);
 629                         if (*buffers != NULL)
 630                                 break;
 631                 }
 632         }
 633
 634         return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
 635 }
 636
 637 static void
 638 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
 639                 uint16_t n, int8_t max_llr_modulus)
 640 {
 641         uint16_t i, byte_idx;
 642
 643         for (i = 0; i < n; ++i) {
 644                 struct rte_mbuf *m = input_ops[i].data;
 645                 while (m != NULL) {
 646                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
 647                                         input_ops[i].offset);
 648                         for (byte_idx = 0; byte_idx < input_ops[i].length;
 649                                         ++byte_idx)
 650                                 llr[byte_idx] = round((double)max_llr_modulus *
 651                                                 llr[byte_idx] / INT8_MAX);
 652
 653                         m = m->next;
 654                 }
 655         }
 656 }
 657
 658 static int
 659 fill_queue_buffers(struct test_op_params *op_params,
 660                 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
 661                 struct rte_mempool *soft_out_mp, uint16_t queue_id,
 662                 const struct rte_bbdev_op_cap *capabilities,
 663                 uint16_t min_alignment, const int socket_id)
 664 {
 665         int ret;
 666         enum op_data_type type;
 667         const uint16_t n = op_params->num_to_process;
 668
 669         struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
 670                 in_mp,
 671                 soft_out_mp,
 672                 hard_out_mp,
 673         };
 674
 675         struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
 676                 &op_params->q_bufs[socket_id][queue_id].inputs,
 677                 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
 678                 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
 679         };
 680
 681         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
 682                 struct op_data_entries *ref_entries =
 683                                 &test_vector.entries[type];
 684                 if (ref_entries->nb_segments == 0)
 685                         continue;
 686
 687                 ret = allocate_buffers_on_socket(queue_ops[type],
 688                                 n * sizeof(struct rte_bbdev_op_data),
 689                                 socket_id);
 690                 TEST_ASSERT_SUCCESS(ret,
 691                                 "Couldn't allocate memory for rte_bbdev_op_data structs");
 692
 693                 ret = init_op_data_objs(*queue_ops[type], ref_entries,
 694                                 mbuf_pools[type], n, type, min_alignment);
 695                 TEST_ASSERT_SUCCESS(ret,
 696                                 "Couldn't init rte_bbdev_op_data structs");
 697         }
 698
 699         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
 700                 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
 701                         capabilities->cap.turbo_dec.max_llr_modulus);
 702
 703         return 0;
 704 }
 705
 706 static void
 707 free_buffers(struct active_device *ad, struct test_op_params *op_params)
 708 {
 709         unsigned int i, j;
 710
 711         rte_mempool_free(ad->ops_mempool);
 712         rte_mempool_free(ad->in_mbuf_pool);
 713         rte_mempool_free(ad->hard_out_mbuf_pool);
 714         rte_mempool_free(ad->soft_out_mbuf_pool);
 715
 716         for (i = 0; i < rte_lcore_count(); ++i) {
 717                 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
 718                         rte_free(op_params->q_bufs[j][i].inputs);
 719                         rte_free(op_params->q_bufs[j][i].hard_outputs);
 720                         rte_free(op_params->q_bufs[j][i].soft_outputs);
 721                 }
 722         }
 723 }
 724
 725 static void
 726 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
 727                 unsigned int start_idx,
 728                 struct rte_bbdev_op_data *inputs,
 729                 struct rte_bbdev_op_data *hard_outputs,
 730                 struct rte_bbdev_op_data *soft_outputs,
 731                 struct rte_bbdev_dec_op *ref_op)
 732 {
 733         unsigned int i;
 734         struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
 735
 736         for (i = 0; i < n; ++i) {
 737                 if (turbo_dec->code_block_mode == 0) {
 738                         ops[i]->turbo_dec.tb_params.ea =
 739                                         turbo_dec->tb_params.ea;
 740                         ops[i]->turbo_dec.tb_params.eb =
 741                                         turbo_dec->tb_params.eb;
 742                         ops[i]->turbo_dec.tb_params.k_pos =
 743                                         turbo_dec->tb_params.k_pos;
 744                         ops[i]->turbo_dec.tb_params.k_neg =
 745                                         turbo_dec->tb_params.k_neg;
 746                         ops[i]->turbo_dec.tb_params.c =
 747                                         turbo_dec->tb_params.c;
 748                         ops[i]->turbo_dec.tb_params.c_neg =
 749                                         turbo_dec->tb_params.c_neg;
 750                         ops[i]->turbo_dec.tb_params.cab =
 751                                         turbo_dec->tb_params.cab;
 752                 } else {
 753                         ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
 754                         ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
 755                 }
 756
 757                 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
 758                 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
 759                 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
 760                 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
 761                 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
 762                 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
 763                 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
 764
 765                 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
 766                 ops[i]->turbo_dec.input = inputs[start_idx + i];
 767                 if (soft_outputs != NULL)
 768                         ops[i]->turbo_dec.soft_output =
 769                                 soft_outputs[start_idx + i];
 770         }
 771 }
 772
 773 static void
 774 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
 775                 unsigned int start_idx,
 776                 struct rte_bbdev_op_data *inputs,
 777                 struct rte_bbdev_op_data *outputs,
 778                 struct rte_bbdev_enc_op *ref_op)
 779 {
 780         unsigned int i;
 781         struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
 782         for (i = 0; i < n; ++i) {
 783                 if (turbo_enc->code_block_mode == 0) {
 784                         ops[i]->turbo_enc.tb_params.ea =
 785                                         turbo_enc->tb_params.ea;
 786                         ops[i]->turbo_enc.tb_params.eb =
 787                                         turbo_enc->tb_params.eb;
 788                         ops[i]->turbo_enc.tb_params.k_pos =
 789                                         turbo_enc->tb_params.k_pos;
 790                         ops[i]->turbo_enc.tb_params.k_neg =
 791                                         turbo_enc->tb_params.k_neg;
 792                         ops[i]->turbo_enc.tb_params.c =
 793                                         turbo_enc->tb_params.c;
 794                         ops[i]->turbo_enc.tb_params.c_neg =
 795                                         turbo_enc->tb_params.c_neg;
 796                         ops[i]->turbo_enc.tb_params.cab =
 797                                         turbo_enc->tb_params.cab;
 798                         ops[i]->turbo_enc.tb_params.ncb_pos =
 799                                         turbo_enc->tb_params.ncb_pos;
 800                         ops[i]->turbo_enc.tb_params.ncb_neg =
 801                                         turbo_enc->tb_params.ncb_neg;
 802                         ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
 803                 } else {
 804                         ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
 805                         ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
 806                         ops[i]->turbo_enc.cb_params.ncb =
 807                                         turbo_enc->cb_params.ncb;
 808                 }
 809                 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
 810                 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
 811                 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
 812
 813                 ops[i]->turbo_enc.output = outputs[start_idx + i];
 814                 ops[i]->turbo_enc.input = inputs[start_idx + i];
 815         }
 816 }
 817
 818 static int
 819 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
 820                 unsigned int order_idx, const int expected_status)
 821 {
 822         TEST_ASSERT(op->status == expected_status,
 823                         "op_status (%d) != expected_status (%d)",
 824                         op->status, expected_status);
 825
 826         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
 827                         "Ordering error, expected %p, got %p",
 828                         (void *)(uintptr_t)order_idx, op->opaque_data);
 829
 830         return TEST_SUCCESS;
 831 }
 832
 833 static int
 834 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
 835                 unsigned int order_idx, const int expected_status)
 836 {
 837         TEST_ASSERT(op->status == expected_status,
 838                         "op_status (%d) != expected_status (%d)",
 839                         op->status, expected_status);
 840
 841         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
 842                         "Ordering error, expected %p, got %p",
 843                         (void *)(uintptr_t)order_idx, op->opaque_data);
 844
 845         return TEST_SUCCESS;
 846 }
 847
 848 static inline int
 849 validate_op_chain(struct rte_bbdev_op_data *op,
 850                 struct op_data_entries *orig_op)
 851 {
 852         uint8_t i;
 853         struct rte_mbuf *m = op->data;
 854         uint8_t nb_dst_segments = orig_op->nb_segments;
 855
 856         TEST_ASSERT(nb_dst_segments == m->nb_segs,
 857                         "Number of segments differ in original (%u) and filled (%u) op",
 858                         nb_dst_segments, m->nb_segs);
 859
 860         for (i = 0; i < nb_dst_segments; ++i) {
 861                 /* Apply offset to the first mbuf segment */
 862                 uint16_t offset = (i == 0) ? op->offset : 0;
 863                 uint16_t data_len = m->data_len - offset;
 864
 865                 TEST_ASSERT(orig_op->segments[i].length == data_len,
 866                                 "Length of segment differ in original (%u) and filled (%u) op",
 867                                 orig_op->segments[i].length, data_len);
 868                 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
 869                                 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
 870                                 data_len,
 871                                 "Output buffers (CB=%u) are not equal", i);
 872                 m = m->next;
 873         }
 874
 875         return TEST_SUCCESS;
 876 }
 877
 878 static int
 879 validate_dec_buffers(struct rte_bbdev_dec_op *ref_op, struct test_buffers *bufs,
 880                 const uint16_t num_to_process)
 881 {
 882         int i;
 883
 884         struct op_data_entries *hard_data_orig =
 885                         &test_vector.entries[DATA_HARD_OUTPUT];
 886         struct op_data_entries *soft_data_orig =
 887                         &test_vector.entries[DATA_SOFT_OUTPUT];
 888
 889         for (i = 0; i < num_to_process; i++) {
 890                 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i],
 891                                 hard_data_orig),
 892                                 "Hard output buffers are not equal");
 893                 if (ref_op->turbo_dec.op_flags &
 894                                 RTE_BBDEV_TURBO_SOFT_OUTPUT)
 895                         TEST_ASSERT_SUCCESS(validate_op_chain(
 896                                         &bufs->soft_outputs[i],
 897                                         soft_data_orig),
 898                                         "Soft output buffers are not equal");
 899         }
 900
 901         return TEST_SUCCESS;
 902 }
 903
 904 static int
 905 validate_enc_buffers(struct test_buffers *bufs, const uint16_t num_to_process)
 906 {
 907         int i;
 908
 909         struct op_data_entries *hard_data_orig =
 910                         &test_vector.entries[DATA_HARD_OUTPUT];
 911
 912         for (i = 0; i < num_to_process; i++)
 913                 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i],
 914                                 hard_data_orig), "");
 915
 916         return TEST_SUCCESS;
 917 }
 918
 919 static int
 920 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
 921                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
 922 {
 923         unsigned int i;
 924         int ret;
 925         struct op_data_entries *hard_data_orig =
 926                         &test_vector.entries[DATA_HARD_OUTPUT];
 927         struct op_data_entries *soft_data_orig =
 928                         &test_vector.entries[DATA_SOFT_OUTPUT];
 929         struct rte_bbdev_op_turbo_dec *ops_td;
 930         struct rte_bbdev_op_data *hard_output;
 931         struct rte_bbdev_op_data *soft_output;
 932         struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
 933
 934         for (i = 0; i < n; ++i) {
 935                 ops_td = &ops[i]->turbo_dec;
 936                 hard_output = &ops_td->hard_output;
 937                 soft_output = &ops_td->soft_output;
 938
 939                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
 940                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
 941                                         "Returned iter_count (%d) > expected iter_count (%d)",
 942                                         ops_td->iter_count, ref_td->iter_count);
 943                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
 944                 TEST_ASSERT_SUCCESS(ret,
 945                                 "Checking status and ordering for decoder failed");
 946
 947                 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
 948                                 hard_data_orig),
 949                                 "Hard output buffers (CB=%u) are not equal",
 950                                 i);
 951
 952                 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
 953                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
 954                                         soft_data_orig),
 955                                         "Soft output buffers (CB=%u) are not equal",
 956                                         i);
 957         }
 958
 959         return TEST_SUCCESS;
 960 }
 961
 962 static int
 963 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
 964                 struct rte_bbdev_enc_op *ref_op)
 965 {
 966         unsigned int i;
 967         int ret;
 968         struct op_data_entries *hard_data_orig =
 969                         &test_vector.entries[DATA_HARD_OUTPUT];
 970
 971         for (i = 0; i < n; ++i) {
 972                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
 973                 TEST_ASSERT_SUCCESS(ret,
 974                                 "Checking status and ordering for encoder failed");
 975                 TEST_ASSERT_SUCCESS(validate_op_chain(
 976                                 &ops[i]->turbo_enc.output,
 977                                 hard_data_orig),
 978                                 "Output buffers (CB=%u) are not equal",
 979                                 i);
 980         }
 981
 982         return TEST_SUCCESS;
 983 }
 984
 985 static void
 986 create_reference_dec_op(struct rte_bbdev_dec_op *op)
 987 {
 988         unsigned int i;
 989         struct op_data_entries *entry;
 990
 991         op->turbo_dec = test_vector.turbo_dec;
 992         entry = &test_vector.entries[DATA_INPUT];
 993         for (i = 0; i < entry->nb_segments; ++i)
 994                 op->turbo_dec.input.length +=
 995                                 entry->segments[i].length;
 996 }
 997
 998 static void
 999 create_reference_enc_op(struct rte_bbdev_enc_op *op)
1000 {
1001         unsigned int i;
1002         struct op_data_entries *entry;
1003
1004         op->turbo_enc = test_vector.turbo_enc;
1005         entry = &test_vector.entries[DATA_INPUT];
1006         for (i = 0; i < entry->nb_segments; ++i)
1007                 op->turbo_enc.input.length +=
1008                                 entry->segments[i].length;
1009 }
1010
1011 static int
1012 init_test_op_params(struct test_op_params *op_params,
1013                 enum rte_bbdev_op_type op_type, const int expected_status,
1014                 const int vector_mask, struct rte_mempool *ops_mp,
1015                 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
1016 {
1017         int ret = 0;
1018         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1019                 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
1020                                 &op_params->ref_dec_op, 1);
1021         else
1022                 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
1023                                 &op_params->ref_enc_op, 1);
1024
1025         TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
1026
1027         op_params->mp = ops_mp;
1028         op_params->burst_sz = burst_sz;
1029         op_params->num_to_process = num_to_process;
1030         op_params->num_lcores = num_lcores;
1031         op_params->vector_mask = vector_mask;
1032         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1033                 op_params->ref_dec_op->status = expected_status;
1034         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
1035                 op_params->ref_enc_op->status = expected_status;
1036
1037         return 0;
1038 }
1039
1040 static int
1041 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
1042                 struct test_op_params *op_params)
1043 {
1044         int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
1045         unsigned int i;
1046         struct active_device *ad;
1047         unsigned int burst_sz = get_burst_sz();
1048         enum rte_bbdev_op_type op_type = test_vector.op_type;
1049         const struct rte_bbdev_op_cap *capabilities = NULL;
1050
1051         ad = &active_devs[dev_id];
1052
1053         /* Check if device supports op_type */
1054         if (!is_avail_op(ad, test_vector.op_type))
1055                 return TEST_SUCCESS;
1056
1057         struct rte_bbdev_info info;
1058         rte_bbdev_info_get(ad->dev_id, &info);
1059         socket_id = GET_SOCKET(info.socket_id);
1060
1061         if (op_type == RTE_BBDEV_OP_NONE)
1062                 op_type = RTE_BBDEV_OP_TURBO_ENC;
1063         f_ret = create_mempools(ad, socket_id, op_type,
1064                         get_num_ops());
1065         if (f_ret != TEST_SUCCESS) {
1066                 printf("Couldn't create mempools");
1067                 goto fail;
1068         }
1069
1070         f_ret = init_test_op_params(op_params, test_vector.op_type,
1071                         test_vector.expected_status,
1072                         test_vector.mask,
1073                         ad->ops_mempool,
1074                         burst_sz,
1075                         get_num_ops(),
1076                         get_num_lcores());
1077         if (f_ret != TEST_SUCCESS) {
1078                 printf("Couldn't init test op params");
1079                 goto fail;
1080         }
1081
1082         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
1083                 /* Find Decoder capabilities */
1084                 const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
1085                 while (cap->type != RTE_BBDEV_OP_NONE) {
1086                         if (cap->type == RTE_BBDEV_OP_TURBO_DEC) {
1087                                 capabilities = cap;
1088                                 break;
1089                         }
1090                 }
1091                 TEST_ASSERT_NOT_NULL(capabilities,
1092                                 "Couldn't find Decoder capabilities");
1093
1094                 create_reference_dec_op(op_params->ref_dec_op);
1095         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1096                 create_reference_enc_op(op_params->ref_enc_op);
1097
1098         for (i = 0; i < ad->nb_queues; ++i) {
1099                 f_ret = fill_queue_buffers(op_params,
1100                                 ad->in_mbuf_pool,
1101                                 ad->hard_out_mbuf_pool,
1102                                 ad->soft_out_mbuf_pool,
1103                                 ad->queue_ids[i],
1104                                 capabilities,
1105                                 info.drv.min_alignment,
1106                                 socket_id);
1107                 if (f_ret != TEST_SUCCESS) {
1108                         printf("Couldn't init queue buffers");
1109                         goto fail;
1110                 }
1111         }
1112
1113         /* Run test case function */
1114         t_ret = test_case_func(ad, op_params);
1115
1116         /* Free active device resources and return */
1117         free_buffers(ad, op_params);
1118         return t_ret;
1119
1120 fail:
1121         free_buffers(ad, op_params);
1122         return TEST_FAILED;
1123 }
1124
1125 /* Run given test function per active device per supported op type
1126  * per burst size.
1127  */
1128 static int
1129 run_test_case(test_case_function *test_case_func)
1130 {
1131         int ret = 0;
1132         uint8_t dev;
1133
1134         /* Alloc op_params */
1135         struct test_op_params *op_params = rte_zmalloc(NULL,
1136                         sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
1137         TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
1138                         RTE_ALIGN(sizeof(struct test_op_params),
1139                                 RTE_CACHE_LINE_SIZE));
1140
1141         /* For each device run test case function */
1142         for (dev = 0; dev < nb_active_devs; ++dev)
1143                 ret |= run_test_case_on_device(test_case_func, dev, op_params);
1144
1145         rte_free(op_params);
1146
1147         return ret;
1148 }
1149
1150 static void
1151 dequeue_event_callback(uint16_t dev_id,
1152                 enum rte_bbdev_event_type event, void *cb_arg,
1153                 void *ret_param)
1154 {
1155         int ret;
1156         uint16_t i;
1157         uint64_t total_time;
1158         uint16_t deq, burst_sz, num_to_process;
1159         uint16_t queue_id = INVALID_QUEUE_ID;
1160         struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
1161         struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
1162         struct test_buffers *bufs;
1163         struct rte_bbdev_info info;
1164
1165         /* Input length in bytes, million operations per second,
1166          * million bits per second.
1167          */
1168         double in_len;
1169
1170         struct thread_params *tp = cb_arg;
1171         RTE_SET_USED(ret_param);
1172         queue_id = tp->queue_id;
1173
1174         /* Find matching thread params using queue_id */
1175         for (i = 0; i < MAX_QUEUES; ++i, ++tp)
1176                 if (tp->queue_id == queue_id)
1177                         break;
1178
1179         if (i == MAX_QUEUES) {
1180                 printf("%s: Queue_id from interrupt details was not found!\n",
1181                                 __func__);
1182                 return;
1183         }
1184
1185         if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
1186                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1187                 printf(
1188                         "Dequeue interrupt handler called for incorrect event!\n");
1189                 return;
1190         }
1191
1192         burst_sz = tp->op_params->burst_sz;
1193         num_to_process = tp->op_params->num_to_process;
1194
1195         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1196                 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, dec_ops,
1197                                 burst_sz);
1198         else
1199                 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, enc_ops,
1200                                 burst_sz);
1201
1202         if (deq < burst_sz) {
1203                 printf(
1204                         "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
1205                         burst_sz, deq);
1206                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1207                 return;
1208         }
1209
1210         if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_to_process) {
1211                 rte_atomic16_add(&tp->nb_dequeued, deq);
1212                 return;
1213         }
1214
1215         total_time = rte_rdtsc_precise() - tp->start_time;
1216
1217         rte_bbdev_info_get(dev_id, &info);
1218
1219         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1220
1221         ret = TEST_SUCCESS;
1222         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1223                 ret = validate_dec_buffers(tp->op_params->ref_dec_op, bufs,
1224                                 num_to_process);
1225         else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1226                 ret = validate_enc_buffers(bufs, num_to_process);
1227
1228         if (ret) {
1229                 printf("Buffers validation failed\n");
1230                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1231         }
1232
1233         switch (test_vector.op_type) {
1234         case RTE_BBDEV_OP_TURBO_DEC:
1235                 in_len = tp->op_params->ref_dec_op->turbo_dec.input.length;
1236                 break;
1237         case RTE_BBDEV_OP_TURBO_ENC:
1238                 in_len = tp->op_params->ref_enc_op->turbo_enc.input.length;
1239                 break;
1240         case RTE_BBDEV_OP_NONE:
1241                 in_len = 0.0;
1242                 break;
1243         default:
1244                 printf("Unknown op type: %d\n", test_vector.op_type);
1245                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1246                 return;
1247         }
1248
1249         tp->mops = ((double)num_to_process / 1000000.0) /
1250                         ((double)total_time / (double)rte_get_tsc_hz());
1251         tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1252                         ((double)total_time / (double)rte_get_tsc_hz());
1253
1254         rte_atomic16_add(&tp->nb_dequeued, deq);
1255 }
1256
1257 static int
1258 throughput_intr_lcore_dec(void *arg)
1259 {
1260         struct thread_params *tp = arg;
1261         unsigned int enqueued;
1262         struct rte_bbdev_dec_op *ops[MAX_BURST];
1263         const uint16_t queue_id = tp->queue_id;
1264         const uint16_t burst_sz = tp->op_params->burst_sz;
1265         const uint16_t num_to_process = tp->op_params->num_to_process;
1266         struct test_buffers *bufs = NULL;
1267         unsigned int allocs_failed = 0;
1268         struct rte_bbdev_info info;
1269         int ret;
1270
1271         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1272                         "BURST_SIZE should be <= %u", MAX_BURST);
1273
1274         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1275                         "Failed to enable interrupts for dev: %u, queue_id: %u",
1276                         tp->dev_id, queue_id);
1277
1278         rte_bbdev_info_get(tp->dev_id, &info);
1279         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1280
1281         rte_atomic16_clear(&tp->processing_status);
1282         rte_atomic16_clear(&tp->nb_dequeued);
1283
1284         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1285                 rte_pause();
1286
1287         tp->start_time = rte_rdtsc_precise();
1288         for (enqueued = 0; enqueued < num_to_process;) {
1289
1290                 uint16_t num_to_enq = burst_sz;
1291
1292                 if (unlikely(num_to_process - enqueued < num_to_enq))
1293                         num_to_enq = num_to_process - enqueued;
1294
1295                 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
1296                                 num_to_enq);
1297                 if (ret != 0) {
1298                         allocs_failed++;
1299                         continue;
1300                 }
1301
1302                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1303                         copy_reference_dec_op(ops, num_to_enq, enqueued,
1304                                         bufs->inputs,
1305                                         bufs->hard_outputs,
1306                                         bufs->soft_outputs,
1307                                         tp->op_params->ref_dec_op);
1308
1309                 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, queue_id, ops,
1310                                 num_to_enq);
1311
1312                 rte_bbdev_dec_op_free_bulk(ops, num_to_enq);
1313         }
1314
1315         if (allocs_failed > 0)
1316                 printf("WARNING: op allocations failed: %u times\n",
1317                                 allocs_failed);
1318
1319         return TEST_SUCCESS;
1320 }
1321
1322 static int
1323 throughput_intr_lcore_enc(void *arg)
1324 {
1325         struct thread_params *tp = arg;
1326         unsigned int enqueued;
1327         struct rte_bbdev_enc_op *ops[MAX_BURST];
1328         const uint16_t queue_id = tp->queue_id;
1329         const uint16_t burst_sz = tp->op_params->burst_sz;
1330         const uint16_t num_to_process = tp->op_params->num_to_process;
1331         struct test_buffers *bufs = NULL;
1332         unsigned int allocs_failed = 0;
1333         struct rte_bbdev_info info;
1334         int ret;
1335
1336         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1337                         "BURST_SIZE should be <= %u", MAX_BURST);
1338
1339         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1340                         "Failed to enable interrupts for dev: %u, queue_id: %u",
1341                         tp->dev_id, queue_id);
1342
1343         rte_bbdev_info_get(tp->dev_id, &info);
1344         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1345
1346         rte_atomic16_clear(&tp->processing_status);
1347         rte_atomic16_clear(&tp->nb_dequeued);
1348
1349         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1350                 rte_pause();
1351
1352         tp->start_time = rte_rdtsc_precise();
1353         for (enqueued = 0; enqueued < num_to_process;) {
1354
1355                 uint16_t num_to_enq = burst_sz;
1356
1357                 if (unlikely(num_to_process - enqueued < num_to_enq))
1358                         num_to_enq = num_to_process - enqueued;
1359
1360                 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
1361                                 num_to_enq);
1362                 if (ret != 0) {
1363                         allocs_failed++;
1364                         continue;
1365                 }
1366
1367                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1368                         copy_reference_enc_op(ops, num_to_enq, enqueued,
1369                                         bufs->inputs,
1370                                         bufs->hard_outputs,
1371                                         tp->op_params->ref_enc_op);
1372
1373                 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, queue_id, ops,
1374                                 num_to_enq);
1375
1376                 rte_bbdev_enc_op_free_bulk(ops, num_to_enq);
1377         }
1378
1379         if (allocs_failed > 0)
1380                 printf("WARNING: op allocations failed: %u times\n",
1381                                 allocs_failed);
1382
1383         return TEST_SUCCESS;
1384 }
1385
1386 static int
1387 throughput_pmd_lcore_dec(void *arg)
1388 {
1389         struct thread_params *tp = arg;
1390         unsigned int enqueued, dequeued;
1391         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1392         uint64_t total_time, start_time;
1393         const uint16_t queue_id = tp->queue_id;
1394         const uint16_t burst_sz = tp->op_params->burst_sz;
1395         const uint16_t num_to_process = tp->op_params->num_to_process;
1396         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1397         struct test_buffers *bufs = NULL;
1398         unsigned int allocs_failed = 0;
1399         int ret;
1400         struct rte_bbdev_info info;
1401
1402         /* Input length in bytes, million operations per second, million bits
1403          * per second.
1404          */
1405         double in_len;
1406
1407         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1408                         "BURST_SIZE should be <= %u", MAX_BURST);
1409
1410         rte_bbdev_info_get(tp->dev_id, &info);
1411         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1412
1413         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1414                 rte_pause();
1415
1416         start_time = rte_rdtsc_precise();
1417         for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) {
1418                 uint16_t deq;
1419
1420                 if (likely(enqueued < num_to_process)) {
1421
1422                         uint16_t num_to_enq = burst_sz;
1423
1424                         if (unlikely(num_to_process - enqueued < num_to_enq))
1425                                 num_to_enq = num_to_process - enqueued;
1426
1427                         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp,
1428                                         ops_enq, num_to_enq);
1429                         if (ret != 0) {
1430                                 allocs_failed++;
1431                                 goto do_dequeue;
1432                         }
1433
1434                         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1435                                 copy_reference_dec_op(ops_enq, num_to_enq,
1436                                                 enqueued,
1437                                                 bufs->inputs,
1438                                                 bufs->hard_outputs,
1439                                                 bufs->soft_outputs,
1440                                                 ref_op);
1441
1442                         enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id,
1443                                         queue_id, ops_enq, num_to_enq);
1444                 }
1445 do_dequeue:
1446                 deq = rte_bbdev_dequeue_dec_ops(tp->dev_id, queue_id, ops_deq,
1447                                 burst_sz);
1448                 dequeued += deq;
1449                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1450         }
1451         total_time = rte_rdtsc_precise() - start_time;
1452
1453         if (allocs_failed > 0)
1454                 printf("WARNING: op allocations failed: %u times\n",
1455                                 allocs_failed);
1456
1457         TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)",
1458                         enqueued, dequeued);
1459
1460         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1461                 ret = validate_dec_buffers(ref_op, bufs, num_to_process);
1462                 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed");
1463         }
1464
1465         in_len = ref_op->turbo_dec.input.length;
1466         tp->mops = ((double)num_to_process / 1000000.0) /
1467                         ((double)total_time / (double)rte_get_tsc_hz());
1468         tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1469                         ((double)total_time / (double)rte_get_tsc_hz());
1470
1471         return TEST_SUCCESS;
1472 }
1473
1474 static int
1475 throughput_pmd_lcore_enc(void *arg)
1476 {
1477         struct thread_params *tp = arg;
1478         unsigned int enqueued, dequeued;
1479         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1480         uint64_t total_time, start_time;
1481         const uint16_t queue_id = tp->queue_id;
1482         const uint16_t burst_sz = tp->op_params->burst_sz;
1483         const uint16_t num_to_process = tp->op_params->num_to_process;
1484         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1485         struct test_buffers *bufs = NULL;
1486         unsigned int allocs_failed = 0;
1487         int ret;
1488         struct rte_bbdev_info info;
1489
1490         /* Input length in bytes, million operations per second, million bits
1491          * per second.
1492          */
1493         double in_len;
1494
1495         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1496                         "BURST_SIZE should be <= %u", MAX_BURST);
1497
1498         rte_bbdev_info_get(tp->dev_id, &info);
1499         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1500
1501         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1502                 rte_pause();
1503
1504         start_time = rte_rdtsc_precise();
1505         for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) {
1506                 uint16_t deq;
1507
1508                 if (likely(enqueued < num_to_process)) {
1509
1510                         uint16_t num_to_enq = burst_sz;
1511
1512                         if (unlikely(num_to_process - enqueued < num_to_enq))
1513                                 num_to_enq = num_to_process - enqueued;
1514
1515                         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp,
1516                                         ops_enq, num_to_enq);
1517                         if (ret != 0) {
1518                                 allocs_failed++;
1519                                 goto do_dequeue;
1520                         }
1521
1522                         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1523                                 copy_reference_enc_op(ops_enq, num_to_enq,
1524                                                 enqueued,
1525                                                 bufs->inputs,
1526                                                 bufs->hard_outputs,
1527                                                 ref_op);
1528
1529                         enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id,
1530                                         queue_id, ops_enq, num_to_enq);
1531                 }
1532 do_dequeue:
1533                 deq = rte_bbdev_dequeue_enc_ops(tp->dev_id, queue_id, ops_deq,
1534                                 burst_sz);
1535                 dequeued += deq;
1536                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1537         }
1538         total_time = rte_rdtsc_precise() - start_time;
1539
1540         if (allocs_failed > 0)
1541                 printf("WARNING: op allocations failed: %u times\n",
1542                                 allocs_failed);
1543
1544         TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)",
1545                         enqueued, dequeued);
1546
1547         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1548                 ret = validate_enc_buffers(bufs, num_to_process);
1549                 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed");
1550         }
1551
1552         in_len = ref_op->turbo_enc.input.length;
1553
1554         tp->mops = ((double)num_to_process / 1000000.0) /
1555                         ((double)total_time / (double)rte_get_tsc_hz());
1556         tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1557                         ((double)total_time / (double)rte_get_tsc_hz());
1558
1559         return TEST_SUCCESS;
1560 }
1561 static void
1562 print_throughput(struct thread_params *t_params, unsigned int used_cores)
1563 {
1564         unsigned int lcore_id, iter = 0;
1565         double total_mops = 0, total_mbps = 0;
1566
1567         RTE_LCORE_FOREACH(lcore_id) {
1568                 if (iter++ >= used_cores)
1569                         break;
1570                 printf("\tlcore_id: %u, throughput: %.8lg MOPS, %.8lg Mbps\n",
1571                 lcore_id, t_params[lcore_id].mops, t_params[lcore_id].mbps);
1572                 total_mops += t_params[lcore_id].mops;
1573                 total_mbps += t_params[lcore_id].mbps;
1574         }
1575         printf(
1576                 "\n\tTotal stats for %u cores: throughput: %.8lg MOPS, %.8lg Mbps\n",
1577                 used_cores, total_mops, total_mbps);
1578 }
1579
1580 /*
1581  * Test function that determines how long an enqueue + dequeue of a burst
1582  * takes on available lcores.
1583  */
1584 static int
1585 throughput_test(struct active_device *ad,
1586                 struct test_op_params *op_params)
1587 {
1588         int ret;
1589         unsigned int lcore_id, used_cores = 0;
1590         struct thread_params t_params[MAX_QUEUES];
1591         struct rte_bbdev_info info;
1592         lcore_function_t *throughput_function;
1593         struct thread_params *tp;
1594         uint16_t num_lcores;
1595         const char *op_type_str;
1596
1597         rte_bbdev_info_get(ad->dev_id, &info);
1598
1599         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
1600         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
1601                         test_vector.op_type);
1602
1603         printf(
1604                 "Throughput test: dev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, int mode: %s, GHz: %lg\n",
1605                         info.dev_name, ad->nb_queues, op_params->burst_sz,
1606                         op_params->num_to_process, op_params->num_lcores,
1607                         op_type_str,
1608                         intr_enabled ? "Interrupt mode" : "PMD mode",
1609                         (double)rte_get_tsc_hz() / 1000000000.0);
1610
1611         /* Set number of lcores */
1612         num_lcores = (ad->nb_queues < (op_params->num_lcores))
1613                         ? ad->nb_queues
1614                         : op_params->num_lcores;
1615
1616         if (intr_enabled) {
1617                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1618                         throughput_function = throughput_intr_lcore_dec;
1619                 else
1620                         throughput_function = throughput_intr_lcore_enc;
1621
1622                 /* Dequeue interrupt callback registration */
1623                 ret = rte_bbdev_callback_register(ad->dev_id,
1624                                 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
1625                                 &t_params);
1626                 if (ret < 0)
1627                         return ret;
1628         } else {
1629                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1630                         throughput_function = throughput_pmd_lcore_dec;
1631                 else
1632                         throughput_function = throughput_pmd_lcore_enc;
1633         }
1634
1635         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
1636
1637         t_params[rte_lcore_id()].dev_id = ad->dev_id;
1638         t_params[rte_lcore_id()].op_params = op_params;
1639         t_params[rte_lcore_id()].queue_id =
1640                         ad->queue_ids[used_cores++];
1641
1642         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1643                 if (used_cores >= num_lcores)
1644                         break;
1645
1646                 t_params[lcore_id].dev_id = ad->dev_id;
1647                 t_params[lcore_id].op_params = op_params;
1648                 t_params[lcore_id].queue_id = ad->queue_ids[used_cores++];
1649
1650                 rte_eal_remote_launch(throughput_function, &t_params[lcore_id],
1651                                 lcore_id);
1652         }
1653
1654         rte_atomic16_set(&op_params->sync, SYNC_START);
1655         ret = throughput_function(&t_params[rte_lcore_id()]);
1656
1657         /* Master core is always used */
1658         used_cores = 1;
1659         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1660                 if (used_cores++ >= num_lcores)
1661                         break;
1662
1663                 ret |= rte_eal_wait_lcore(lcore_id);
1664         }
1665
1666         /* Return if test failed */
1667         if (ret)
1668                 return ret;
1669
1670         /* Print throughput if interrupts are disabled and test passed */
1671         if (!intr_enabled) {
1672                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1673                         print_throughput(t_params, num_lcores);
1674                 return ret;
1675         }
1676
1677         /* In interrupt TC we need to wait for the interrupt callback to deqeue
1678          * all pending operations. Skip waiting for queues which reported an
1679          * error using processing_status variable.
1680          * Wait for master lcore operations.
1681          */
1682         tp = &t_params[rte_lcore_id()];
1683         while ((rte_atomic16_read(&tp->nb_dequeued) <
1684                         op_params->num_to_process) &&
1685                         (rte_atomic16_read(&tp->processing_status) !=
1686                         TEST_FAILED))
1687                 rte_pause();
1688
1689         ret |= rte_atomic16_read(&tp->processing_status);
1690
1691         /* Wait for slave lcores operations */
1692         used_cores = 1;
1693         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1694                 tp = &t_params[lcore_id];
1695                 if (used_cores++ >= num_lcores)
1696                         break;
1697
1698                 while ((rte_atomic16_read(&tp->nb_dequeued) <
1699                                 op_params->num_to_process) &&
1700                                 (rte_atomic16_read(&tp->processing_status) !=
1701                                 TEST_FAILED))
1702                         rte_pause();
1703
1704                 ret |= rte_atomic16_read(&tp->processing_status);
1705         }
1706
1707         /* Print throughput if test passed */
1708         if (!ret && test_vector.op_type != RTE_BBDEV_OP_NONE)
1709                 print_throughput(t_params, num_lcores);
1710
1711         return ret;
1712 }
1713
1714 static int
1715 latency_test_dec(struct rte_mempool *mempool,
1716                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
1717                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
1718                 const uint16_t num_to_process, uint16_t burst_sz,
1719                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
1720 {
1721         int ret = TEST_SUCCESS;
1722         uint16_t i, j, dequeued;
1723         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1724         uint64_t start_time = 0, last_time = 0;
1725
1726         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1727                 uint16_t enq = 0, deq = 0;
1728                 bool first_time = true;
1729                 last_time = 0;
1730
1731                 if (unlikely(num_to_process - dequeued < burst_sz))
1732                         burst_sz = num_to_process - dequeued;
1733
1734                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1735                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1736                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1737                                         bufs->inputs,
1738                                         bufs->hard_outputs,
1739                                         bufs->soft_outputs,
1740                                         ref_op);
1741
1742                 /* Set counter to validate the ordering */
1743                 for (j = 0; j < burst_sz; ++j)
1744                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1745
1746                 start_time = rte_rdtsc_precise();
1747
1748                 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
1749                                 burst_sz);
1750                 TEST_ASSERT(enq == burst_sz,
1751                                 "Error enqueueing burst, expected %u, got %u",
1752                                 burst_sz, enq);
1753
1754                 /* Dequeue */
1755                 do {
1756                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1757                                         &ops_deq[deq], burst_sz - deq);
1758                         if (likely(first_time && (deq > 0))) {
1759                                 last_time = rte_rdtsc_precise() - start_time;
1760                                 first_time = false;
1761                         }
1762                 } while (unlikely(burst_sz != deq));
1763
1764                 *max_time = RTE_MAX(*max_time, last_time);
1765                 *min_time = RTE_MIN(*min_time, last_time);
1766                 *total_time += last_time;
1767
1768                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1769                         ret = validate_dec_op(ops_deq, burst_sz, ref_op,
1770                                         vector_mask);
1771                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1772                 }
1773
1774                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1775                 dequeued += deq;
1776         }
1777
1778         return i;
1779 }
1780
1781 static int
1782 latency_test_enc(struct rte_mempool *mempool,
1783                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
1784                 uint16_t dev_id, uint16_t queue_id,
1785                 const uint16_t num_to_process, uint16_t burst_sz,
1786                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
1787 {
1788         int ret = TEST_SUCCESS;
1789         uint16_t i, j, dequeued;
1790         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1791         uint64_t start_time = 0, last_time = 0;
1792
1793         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1794                 uint16_t enq = 0, deq = 0;
1795                 bool first_time = true;
1796                 last_time = 0;
1797
1798                 if (unlikely(num_to_process - dequeued < burst_sz))
1799                         burst_sz = num_to_process - dequeued;
1800
1801                 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
1802                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1803                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
1804                                         bufs->inputs,
1805                                         bufs->hard_outputs,
1806                                         ref_op);
1807
1808                 /* Set counter to validate the ordering */
1809                 for (j = 0; j < burst_sz; ++j)
1810                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1811
1812                 start_time = rte_rdtsc_precise();
1813
1814                 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
1815                                 burst_sz);
1816                 TEST_ASSERT(enq == burst_sz,
1817                                 "Error enqueueing burst, expected %u, got %u",
1818                                 burst_sz, enq);
1819
1820                 /* Dequeue */
1821                 do {
1822                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1823                                         &ops_deq[deq], burst_sz - deq);
1824                         if (likely(first_time && (deq > 0))) {
1825                                 last_time += rte_rdtsc_precise() - start_time;
1826                                 first_time = false;
1827                         }
1828                 } while (unlikely(burst_sz != deq));
1829
1830                 *max_time = RTE_MAX(*max_time, last_time);
1831                 *min_time = RTE_MIN(*min_time, last_time);
1832                 *total_time += last_time;
1833
1834                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1835                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
1836                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1837                 }
1838
1839                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1840                 dequeued += deq;
1841         }
1842
1843         return i;
1844 }
1845
1846 static int
1847 latency_test(struct active_device *ad,
1848                 struct test_op_params *op_params)
1849 {
1850         int iter;
1851         uint16_t burst_sz = op_params->burst_sz;
1852         const uint16_t num_to_process = op_params->num_to_process;
1853         const enum rte_bbdev_op_type op_type = test_vector.op_type;
1854         const uint16_t queue_id = ad->queue_ids[0];
1855         struct test_buffers *bufs = NULL;
1856         struct rte_bbdev_info info;
1857         uint64_t total_time, min_time, max_time;
1858         const char *op_type_str;
1859
1860         total_time = max_time = 0;
1861         min_time = UINT64_MAX;
1862
1863         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1864                         "BURST_SIZE should be <= %u", MAX_BURST);
1865
1866         rte_bbdev_info_get(ad->dev_id, &info);
1867         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1868
1869         op_type_str = rte_bbdev_op_type_str(op_type);
1870         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
1871
1872         printf(
1873                 "Validation/Latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
1874                         info.dev_name, burst_sz, num_to_process, op_type_str);
1875
1876         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1877                 iter = latency_test_dec(op_params->mp, bufs,
1878                                 op_params->ref_dec_op, op_params->vector_mask,
1879                                 ad->dev_id, queue_id, num_to_process,
1880                                 burst_sz, &total_time, &min_time, &max_time);
1881         else
1882                 iter = latency_test_enc(op_params->mp, bufs,
1883                                 op_params->ref_enc_op, ad->dev_id, queue_id,
1884                                 num_to_process, burst_sz, &total_time,
1885                                 &min_time, &max_time);
1886
1887         if (iter <= 0)
1888                 return TEST_FAILED;
1889
1890         printf("\toperation latency:\n"
1891                         "\t\tavg latency: %lg cycles, %lg us\n"
1892                         "\t\tmin latency: %lg cycles, %lg us\n"
1893                         "\t\tmax latency: %lg cycles, %lg us\n",
1894                         (double)total_time / (double)iter,
1895                         (double)(total_time * 1000000) / (double)iter /
1896                         (double)rte_get_tsc_hz(), (double)min_time,
1897                         (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
1898                         (double)max_time, (double)(max_time * 1000000) /
1899                         (double)rte_get_tsc_hz());
1900
1901         return TEST_SUCCESS;
1902 }
1903
1904 #ifdef RTE_BBDEV_OFFLOAD_COST
1905 static int
1906 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
1907                 struct rte_bbdev_stats *stats)
1908 {
1909         struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
1910         struct rte_bbdev_stats *q_stats;
1911
1912         if (queue_id >= dev->data->num_queues)
1913                 return -1;
1914
1915         q_stats = &dev->data->queues[queue_id].queue_stats;
1916
1917         stats->enqueued_count = q_stats->enqueued_count;
1918         stats->dequeued_count = q_stats->dequeued_count;
1919         stats->enqueue_err_count = q_stats->enqueue_err_count;
1920         stats->dequeue_err_count = q_stats->dequeue_err_count;
1921         stats->offload_time = q_stats->offload_time;
1922
1923         return 0;
1924 }
1925
1926 static int
1927 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
1928                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
1929                 uint16_t queue_id, const uint16_t num_to_process,
1930                 uint16_t burst_sz, struct test_time_stats *time_st)
1931 {
1932         int i, dequeued, ret;
1933         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1934         uint64_t enq_start_time, deq_start_time;
1935         uint64_t enq_sw_last_time, deq_last_time;
1936         struct rte_bbdev_stats stats;
1937
1938         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1939                 uint16_t enq = 0, deq = 0;
1940
1941                 if (unlikely(num_to_process - dequeued < burst_sz))
1942                         burst_sz = num_to_process - dequeued;
1943
1944                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1945                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1946                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1947                                         bufs->inputs,
1948                                         bufs->hard_outputs,
1949                                         bufs->soft_outputs,
1950                                         ref_op);
1951
1952                 /* Start time meas for enqueue function offload latency */
1953                 enq_start_time = rte_rdtsc_precise();
1954                 do {
1955                         enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
1956                                         &ops_enq[enq], burst_sz - enq);
1957                 } while (unlikely(burst_sz != enq));
1958
1959                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
1960                 TEST_ASSERT_SUCCESS(ret,
1961                                 "Failed to get stats for queue (%u) of device (%u)",
1962                                 queue_id, dev_id);
1963
1964                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
1965                                 stats.offload_time;
1966                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
1967                                 enq_sw_last_time);
1968                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
1969                                 enq_sw_last_time);
1970                 time_st->enq_sw_tot_time += enq_sw_last_time;
1971
1972                 time_st->enq_tur_max_time = RTE_MAX(time_st->enq_tur_max_time,
1973                                 stats.offload_time);
1974                 time_st->enq_tur_min_time = RTE_MIN(time_st->enq_tur_min_time,
1975                                 stats.offload_time);
1976                 time_st->enq_tur_tot_time += stats.offload_time;
1977
1978                 /* ensure enqueue has been completed */
1979                 rte_delay_ms(10);
1980
1981                 /* Start time meas for dequeue function offload latency */
1982                 deq_start_time = rte_rdtsc_precise();
1983                 /* Dequeue one operation */
1984                 do {
1985                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1986                                         &ops_deq[deq], 1);
1987                 } while (unlikely(deq != 1));
1988
1989                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
1990                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
1991                                 deq_last_time);
1992                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
1993                                 deq_last_time);
1994                 time_st->deq_tot_time += deq_last_time;
1995
1996                 /* Dequeue remaining operations if needed*/
1997                 while (burst_sz != deq)
1998                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1999                                         &ops_deq[deq], burst_sz - deq);
2000
2001                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
2002                 dequeued += deq;
2003         }
2004
2005         return i;
2006 }
2007
2008 static int
2009 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
2010                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
2011                 uint16_t queue_id, const uint16_t num_to_process,
2012                 uint16_t burst_sz, struct test_time_stats *time_st)
2013 {
2014         int i, dequeued, ret;
2015         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2016         uint64_t enq_start_time, deq_start_time;
2017         uint64_t enq_sw_last_time, deq_last_time;
2018         struct rte_bbdev_stats stats;
2019
2020         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2021                 uint16_t enq = 0, deq = 0;
2022
2023                 if (unlikely(num_to_process - dequeued < burst_sz))
2024                         burst_sz = num_to_process - dequeued;
2025
2026                 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
2027                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2028                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
2029                                         bufs->inputs,
2030                                         bufs->hard_outputs,
2031                                         ref_op);
2032
2033                 /* Start time meas for enqueue function offload latency */
2034                 enq_start_time = rte_rdtsc_precise();
2035                 do {
2036                         enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
2037                                         &ops_enq[enq], burst_sz - enq);
2038                 } while (unlikely(burst_sz != enq));
2039
2040                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
2041                 TEST_ASSERT_SUCCESS(ret,
2042                                 "Failed to get stats for queue (%u) of device (%u)",
2043                                 queue_id, dev_id);
2044
2045                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
2046                                 stats.offload_time;
2047                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
2048                                 enq_sw_last_time);
2049                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
2050                                 enq_sw_last_time);
2051                 time_st->enq_sw_tot_time += enq_sw_last_time;
2052
2053                 time_st->enq_tur_max_time = RTE_MAX(time_st->enq_tur_max_time,
2054                                 stats.offload_time);
2055                 time_st->enq_tur_min_time = RTE_MIN(time_st->enq_tur_min_time,
2056                                 stats.offload_time);
2057                 time_st->enq_tur_tot_time += stats.offload_time;
2058
2059                 /* ensure enqueue has been completed */
2060                 rte_delay_ms(10);
2061
2062                 /* Start time meas for dequeue function offload latency */
2063                 deq_start_time = rte_rdtsc_precise();
2064                 /* Dequeue one operation */
2065                 do {
2066                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2067                                         &ops_deq[deq], 1);
2068                 } while (unlikely(deq != 1));
2069
2070                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
2071                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
2072                                 deq_last_time);
2073                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
2074                                 deq_last_time);
2075                 time_st->deq_tot_time += deq_last_time;
2076
2077                 while (burst_sz != deq)
2078                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2079                                         &ops_deq[deq], burst_sz - deq);
2080
2081                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
2082                 dequeued += deq;
2083         }
2084
2085         return i;
2086 }
2087 #endif
2088
2089 static int
2090 offload_cost_test(struct active_device *ad,
2091                 struct test_op_params *op_params)
2092 {
2093 #ifndef RTE_BBDEV_OFFLOAD_COST
2094         RTE_SET_USED(ad);
2095         RTE_SET_USED(op_params);
2096         printf("Offload latency test is disabled.\n");
2097         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
2098         return TEST_SKIPPED;
2099 #else
2100         int iter;
2101         uint16_t burst_sz = op_params->burst_sz;
2102         const uint16_t num_to_process = op_params->num_to_process;
2103         const enum rte_bbdev_op_type op_type = test_vector.op_type;
2104         const uint16_t queue_id = ad->queue_ids[0];
2105         struct test_buffers *bufs = NULL;
2106         struct rte_bbdev_info info;
2107         const char *op_type_str;
2108         struct test_time_stats time_st;
2109
2110         memset(&time_st, 0, sizeof(struct test_time_stats));
2111         time_st.enq_sw_min_time = UINT64_MAX;
2112         time_st.enq_tur_min_time = UINT64_MAX;
2113         time_st.deq_min_time = UINT64_MAX;
2114
2115         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2116                         "BURST_SIZE should be <= %u", MAX_BURST);
2117
2118         rte_bbdev_info_get(ad->dev_id, &info);
2119         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2120
2121         op_type_str = rte_bbdev_op_type_str(op_type);
2122         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2123
2124         printf(
2125                 "Offload latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
2126                         info.dev_name, burst_sz, num_to_process, op_type_str);
2127
2128         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2129                 iter = offload_latency_test_dec(op_params->mp, bufs,
2130                                 op_params->ref_dec_op, ad->dev_id, queue_id,
2131                                 num_to_process, burst_sz, &time_st);
2132         else
2133                 iter = offload_latency_test_enc(op_params->mp, bufs,
2134                                 op_params->ref_enc_op, ad->dev_id, queue_id,
2135                                 num_to_process, burst_sz, &time_st);
2136
2137         if (iter <= 0)
2138                 return TEST_FAILED;
2139
2140         printf("\tenq offload cost latency:\n"
2141                         "\t\tsoftware avg %lg cycles, %lg us\n"
2142                         "\t\tsoftware min %lg cycles, %lg us\n"
2143                         "\t\tsoftware max %lg cycles, %lg us\n"
2144                         "\t\tturbo avg %lg cycles, %lg us\n"
2145                         "\t\tturbo min %lg cycles, %lg us\n"
2146                         "\t\tturbo max %lg cycles, %lg us\n",
2147                         (double)time_st.enq_sw_tot_time / (double)iter,
2148                         (double)(time_st.enq_sw_tot_time * 1000000) /
2149                         (double)iter / (double)rte_get_tsc_hz(),
2150                         (double)time_st.enq_sw_min_time,
2151                         (double)(time_st.enq_sw_min_time * 1000000) /
2152                         rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
2153                         (double)(time_st.enq_sw_max_time * 1000000) /
2154                         rte_get_tsc_hz(), (double)time_st.enq_tur_tot_time /
2155                         (double)iter,
2156                         (double)(time_st.enq_tur_tot_time * 1000000) /
2157                         (double)iter / (double)rte_get_tsc_hz(),
2158                         (double)time_st.enq_tur_min_time,
2159                         (double)(time_st.enq_tur_min_time * 1000000) /
2160                         rte_get_tsc_hz(), (double)time_st.enq_tur_max_time,
2161                         (double)(time_st.enq_tur_max_time * 1000000) /
2162                         rte_get_tsc_hz());
2163
2164         printf("\tdeq offload cost latency - one op:\n"
2165                         "\t\tavg %lg cycles, %lg us\n"
2166                         "\t\tmin %lg cycles, %lg us\n"
2167                         "\t\tmax %lg cycles, %lg us\n",
2168                         (double)time_st.deq_tot_time / (double)iter,
2169                         (double)(time_st.deq_tot_time * 1000000) /
2170                         (double)iter / (double)rte_get_tsc_hz(),
2171                         (double)time_st.deq_min_time,
2172                         (double)(time_st.deq_min_time * 1000000) /
2173                         rte_get_tsc_hz(), (double)time_st.deq_max_time,
2174                         (double)(time_st.deq_max_time * 1000000) /
2175                         rte_get_tsc_hz());
2176
2177         return TEST_SUCCESS;
2178 #endif
2179 }
2180
2181 #ifdef RTE_BBDEV_OFFLOAD_COST
2182 static int
2183 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
2184                 const uint16_t num_to_process, uint16_t burst_sz,
2185                 uint64_t *deq_tot_time, uint64_t *deq_min_time,
2186                 uint64_t *deq_max_time)
2187 {
2188         int i, deq_total;
2189         struct rte_bbdev_dec_op *ops[MAX_BURST];
2190         uint64_t deq_start_time, deq_last_time;
2191
2192         /* Test deq offload latency from an empty queue */
2193
2194         for (i = 0, deq_total = 0; deq_total < num_to_process;
2195                         ++i, deq_total += burst_sz) {
2196                 deq_start_time = rte_rdtsc_precise();
2197
2198                 if (unlikely(num_to_process - deq_total < burst_sz))
2199                         burst_sz = num_to_process - deq_total;
2200                 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz);
2201
2202                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
2203                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
2204                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
2205                 *deq_tot_time += deq_last_time;
2206         }
2207
2208         return i;
2209 }
2210
2211 static int
2212 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
2213                 const uint16_t num_to_process, uint16_t burst_sz,
2214                 uint64_t *deq_tot_time, uint64_t *deq_min_time,
2215                 uint64_t *deq_max_time)
2216 {
2217         int i, deq_total;
2218         struct rte_bbdev_enc_op *ops[MAX_BURST];
2219         uint64_t deq_start_time, deq_last_time;
2220
2221         /* Test deq offload latency from an empty queue */
2222         for (i = 0, deq_total = 0; deq_total < num_to_process;
2223                         ++i, deq_total += burst_sz) {
2224                 deq_start_time = rte_rdtsc_precise();
2225
2226                 if (unlikely(num_to_process - deq_total < burst_sz))
2227                         burst_sz = num_to_process - deq_total;
2228                 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz);
2229
2230                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
2231                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
2232                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
2233                 *deq_tot_time += deq_last_time;
2234         }
2235
2236         return i;
2237 }
2238 #endif
2239
2240 static int
2241 offload_latency_empty_q_test(struct active_device *ad,
2242                 struct test_op_params *op_params)
2243 {
2244 #ifndef RTE_BBDEV_OFFLOAD_COST
2245         RTE_SET_USED(ad);
2246         RTE_SET_USED(op_params);
2247         printf("Offload latency empty dequeue test is disabled.\n");
2248         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
2249         return TEST_SKIPPED;
2250 #else
2251         int iter;
2252         uint64_t deq_tot_time, deq_min_time, deq_max_time;
2253         uint16_t burst_sz = op_params->burst_sz;
2254         const uint16_t num_to_process = op_params->num_to_process;
2255         const enum rte_bbdev_op_type op_type = test_vector.op_type;
2256         const uint16_t queue_id = ad->queue_ids[0];
2257         struct rte_bbdev_info info;
2258         const char *op_type_str;
2259
2260         deq_tot_time = deq_max_time = 0;
2261         deq_min_time = UINT64_MAX;
2262
2263         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2264                         "BURST_SIZE should be <= %u", MAX_BURST);
2265
2266         rte_bbdev_info_get(ad->dev_id, &info);
2267
2268         op_type_str = rte_bbdev_op_type_str(op_type);
2269         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2270
2271         printf(
2272                 "Offload latency empty dequeue test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
2273                         info.dev_name, burst_sz, num_to_process, op_type_str);
2274
2275         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2276                 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
2277                                 num_to_process, burst_sz, &deq_tot_time,
2278                                 &deq_min_time, &deq_max_time);
2279         else
2280                 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
2281                                 num_to_process, burst_sz, &deq_tot_time,
2282                                 &deq_min_time, &deq_max_time);
2283
2284         if (iter <= 0)
2285                 return TEST_FAILED;
2286
2287         printf("\tempty deq offload\n"
2288                         "\t\tavg. latency: %lg cycles, %lg us\n"
2289                         "\t\tmin. latency: %lg cycles, %lg us\n"
2290                         "\t\tmax. latency: %lg cycles, %lg us\n",
2291                         (double)deq_tot_time / (double)iter,
2292                         (double)(deq_tot_time * 1000000) / (double)iter /
2293                         (double)rte_get_tsc_hz(), (double)deq_min_time,
2294                         (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
2295                         (double)deq_max_time, (double)(deq_max_time * 1000000) /
2296                         rte_get_tsc_hz());
2297
2298         return TEST_SUCCESS;
2299 #endif
2300 }
2301
2302 static int
2303 throughput_tc(void)
2304 {
2305         return run_test_case(throughput_test);
2306 }
2307
2308 static int
2309 offload_cost_tc(void)
2310 {
2311         return run_test_case(offload_cost_test);
2312 }
2313
2314 static int
2315 offload_latency_empty_q_tc(void)
2316 {
2317         return run_test_case(offload_latency_empty_q_test);
2318 }
2319
2320 static int
2321 latency_tc(void)
2322 {
2323         return run_test_case(latency_test);
2324 }
2325
2326 static int
2327 interrupt_tc(void)
2328 {
2329         return run_test_case(throughput_test);
2330 }
2331
2332 static struct unit_test_suite bbdev_throughput_testsuite = {
2333         .suite_name = "BBdev Throughput Tests",
2334         .setup = testsuite_setup,
2335         .teardown = testsuite_teardown,
2336         .unit_test_cases = {
2337                 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
2338                 TEST_CASES_END() /**< NULL terminate unit test array */
2339         }
2340 };
2341
2342 static struct unit_test_suite bbdev_validation_testsuite = {
2343         .suite_name = "BBdev Validation Tests",
2344         .setup = testsuite_setup,
2345         .teardown = testsuite_teardown,
2346         .unit_test_cases = {
2347                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
2348                 TEST_CASES_END() /**< NULL terminate unit test array */
2349         }
2350 };
2351
2352 static struct unit_test_suite bbdev_latency_testsuite = {
2353         .suite_name = "BBdev Latency Tests",
2354         .setup = testsuite_setup,
2355         .teardown = testsuite_teardown,
2356         .unit_test_cases = {
2357                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
2358                 TEST_CASES_END() /**< NULL terminate unit test array */
2359         }
2360 };
2361
2362 static struct unit_test_suite bbdev_offload_cost_testsuite = {
2363         .suite_name = "BBdev Offload Cost Tests",
2364         .setup = testsuite_setup,
2365         .teardown = testsuite_teardown,
2366         .unit_test_cases = {
2367                 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
2368                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
2369                 TEST_CASES_END() /**< NULL terminate unit test array */
2370         }
2371 };
2372
2373 static struct unit_test_suite bbdev_interrupt_testsuite = {
2374         .suite_name = "BBdev Interrupt Tests",
2375         .setup = interrupt_testsuite_setup,
2376         .teardown = testsuite_teardown,
2377         .unit_test_cases = {
2378                 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
2379                 TEST_CASES_END() /**< NULL terminate unit test array */
2380         }
2381 };
2382
2383 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
2384 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
2385 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
2386 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
2387 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);