app/test-bbdev/test_bbdev_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7
   8 #include <rte_eal.h>
   9 #include <rte_common.h>
  10 #include <rte_dev.h>
  11 #include <rte_launch.h>
  12 #include <rte_bbdev.h>
  13 #include <rte_cycles.h>
  14 #include <rte_lcore.h>
  15 #include <rte_malloc.h>
  16 #include <rte_random.h>
  17 #include <rte_hexdump.h>
  18
  19 #include "main.h"
  20 #include "test_bbdev_vector.h"
  21
  22 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
  23
  24 #define MAX_QUEUES RTE_MAX_LCORE
  25
  26 #define OPS_CACHE_SIZE 256U
  27 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
  28
  29 #define SYNC_WAIT 0
  30 #define SYNC_START 1
  31
  32 #define INVALID_QUEUE_ID -1
  33
  34 static struct test_bbdev_vector test_vector;
  35
  36 /* Switch between PMD and Interrupt for throughput TC */
  37 static bool intr_enabled;
  38
  39 /* Represents tested active devices */
  40 static struct active_device {
  41         const char *driver_name;
  42         uint8_t dev_id;
  43         uint16_t supported_ops;
  44         uint16_t queue_ids[MAX_QUEUES];
  45         uint16_t nb_queues;
  46         struct rte_mempool *ops_mempool;
  47         struct rte_mempool *in_mbuf_pool;
  48         struct rte_mempool *hard_out_mbuf_pool;
  49         struct rte_mempool *soft_out_mbuf_pool;
  50 } active_devs[RTE_BBDEV_MAX_DEVS];
  51
  52 static uint8_t nb_active_devs;
  53
  54 /* Data buffers used by BBDEV ops */
  55 struct test_buffers {
  56         struct rte_bbdev_op_data *inputs;
  57         struct rte_bbdev_op_data *hard_outputs;
  58         struct rte_bbdev_op_data *soft_outputs;
  59 };
  60
  61 /* Operation parameters specific for given test case */
  62 struct test_op_params {
  63         struct rte_mempool *mp;
  64         struct rte_bbdev_dec_op *ref_dec_op;
  65         struct rte_bbdev_enc_op *ref_enc_op;
  66         uint16_t burst_sz;
  67         uint16_t num_to_process;
  68         uint16_t num_lcores;
  69         int vector_mask;
  70         rte_atomic16_t sync;
  71         struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
  72 };
  73
  74 /* Contains per lcore params */
  75 struct thread_params {
  76         uint8_t dev_id;
  77         uint16_t queue_id;
  78         uint64_t start_time;
  79         double mops;
  80         double mbps;
  81         rte_atomic16_t nb_dequeued;
  82         rte_atomic16_t processing_status;
  83         struct test_op_params *op_params;
  84 };
  85
  86 typedef int (test_case_function)(struct active_device *ad,
  87                 struct test_op_params *op_params);
  88
  89 static inline void
  90 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
  91 {
  92         ad->supported_ops |= (1 << op_type);
  93 }
  94
  95 static inline bool
  96 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
  97 {
  98         return ad->supported_ops & (1 << op_type);
  99 }
 100
 101 static inline bool
 102 flags_match(uint32_t flags_req, uint32_t flags_present)
 103 {
 104         return (flags_req & flags_present) == flags_req;
 105 }
 106
 107 static void
 108 clear_soft_out_cap(uint32_t *op_flags)
 109 {
 110         *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
 111         *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
 112         *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
 113 }
 114
 115 static int
 116 check_dev_cap(const struct rte_bbdev_info *dev_info)
 117 {
 118         unsigned int i;
 119         unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs;
 120         const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
 121
 122         nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
 123         nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
 124         nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
 125
 126         for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
 127                 if (op_cap->type != test_vector.op_type)
 128                         continue;
 129
 130                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
 131                         const struct rte_bbdev_op_cap_turbo_dec *cap =
 132                                         &op_cap->cap.turbo_dec;
 133                         /* Ignore lack of soft output capability, just skip
 134                          * checking if soft output is valid.
 135                          */
 136                         if ((test_vector.turbo_dec.op_flags &
 137                                         RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
 138                                         !(cap->capability_flags &
 139                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 140                                 printf(
 141                                         "WARNING: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
 142                                         dev_info->dev_name);
 143                                 clear_soft_out_cap(
 144                                         &test_vector.turbo_dec.op_flags);
 145                         }
 146
 147                         if (!flags_match(test_vector.turbo_dec.op_flags,
 148                                         cap->capability_flags))
 149                                 return TEST_FAILED;
 150                         if (nb_inputs > cap->num_buffers_src) {
 151                                 printf("Too many inputs defined: %u, max: %u\n",
 152                                         nb_inputs, cap->num_buffers_src);
 153                                 return TEST_FAILED;
 154                         }
 155                         if (nb_soft_outputs > cap->num_buffers_soft_out &&
 156                                         (test_vector.turbo_dec.op_flags &
 157                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 158                                 printf(
 159                                         "Too many soft outputs defined: %u, max: %u\n",
 160                                                 nb_soft_outputs,
 161                                                 cap->num_buffers_soft_out);
 162                                 return TEST_FAILED;
 163                         }
 164                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 165                                 printf(
 166                                         "Too many hard outputs defined: %u, max: %u\n",
 167                                                 nb_hard_outputs,
 168                                                 cap->num_buffers_hard_out);
 169                                 return TEST_FAILED;
 170                         }
 171                         if (intr_enabled && !(cap->capability_flags &
 172                                         RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
 173                                 printf(
 174                                         "Dequeue interrupts are not supported!\n");
 175                                 return TEST_FAILED;
 176                         }
 177
 178                         return TEST_SUCCESS;
 179                 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
 180                         const struct rte_bbdev_op_cap_turbo_enc *cap =
 181                                         &op_cap->cap.turbo_enc;
 182
 183                         if (!flags_match(test_vector.turbo_enc.op_flags,
 184                                         cap->capability_flags))
 185                                 return TEST_FAILED;
 186                         if (nb_inputs > cap->num_buffers_src) {
 187                                 printf("Too many inputs defined: %u, max: %u\n",
 188                                         nb_inputs, cap->num_buffers_src);
 189                                 return TEST_FAILED;
 190                         }
 191                         if (nb_hard_outputs > cap->num_buffers_dst) {
 192                                 printf(
 193                                         "Too many hard outputs defined: %u, max: %u\n",
 194                                         nb_hard_outputs, cap->num_buffers_src);
 195                                 return TEST_FAILED;
 196                         }
 197                         if (intr_enabled && !(cap->capability_flags &
 198                                         RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
 199                                 printf(
 200                                         "Dequeue interrupts are not supported!\n");
 201                                 return TEST_FAILED;
 202                         }
 203
 204                         return TEST_SUCCESS;
 205                 }
 206         }
 207
 208         if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
 209                 return TEST_SUCCESS; /* Special case for NULL device */
 210
 211         return TEST_FAILED;
 212 }
 213
 214 /* calculates optimal mempool size not smaller than the val */
 215 static unsigned int
 216 optimal_mempool_size(unsigned int val)
 217 {
 218         return rte_align32pow2(val + 1) - 1;
 219 }
 220
 221 /* allocates mbuf mempool for inputs and outputs */
 222 static struct rte_mempool *
 223 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
 224                 int socket_id, unsigned int mbuf_pool_size,
 225                 const char *op_type_str)
 226 {
 227         unsigned int i;
 228         uint32_t max_seg_sz = 0;
 229         char pool_name[RTE_MEMPOOL_NAMESIZE];
 230
 231         /* find max input segment size */
 232         for (i = 0; i < entries->nb_segments; ++i)
 233                 if (entries->segments[i].length > max_seg_sz)
 234                         max_seg_sz = entries->segments[i].length;
 235
 236         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 237                         dev_id);
 238         return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
 239                         RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM,
 240                         (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
 241 }
 242
 243 static int
 244 create_mempools(struct active_device *ad, int socket_id,
 245                 enum rte_bbdev_op_type op_type, uint16_t num_ops)
 246 {
 247         struct rte_mempool *mp;
 248         unsigned int ops_pool_size, mbuf_pool_size = 0;
 249         char pool_name[RTE_MEMPOOL_NAMESIZE];
 250         const char *op_type_str;
 251
 252         struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
 253         struct op_data_entries *hard_out =
 254                         &test_vector.entries[DATA_HARD_OUTPUT];
 255         struct op_data_entries *soft_out =
 256                         &test_vector.entries[DATA_SOFT_OUTPUT];
 257
 258         /* allocate ops mempool */
 259         ops_pool_size = optimal_mempool_size(RTE_MAX(
 260                         /* Ops used plus 1 reference op */
 261                         RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
 262                         /* Minimal cache size plus 1 reference op */
 263                         (unsigned int)(1.5 * rte_lcore_count() *
 264                                         OPS_CACHE_SIZE + 1)),
 265                         OPS_POOL_SIZE_MIN));
 266
 267         op_type_str = rte_bbdev_op_type_str(op_type);
 268         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
 269
 270         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 271                         ad->dev_id);
 272         mp = rte_bbdev_op_pool_create(pool_name, op_type,
 273                         ops_pool_size, OPS_CACHE_SIZE, socket_id);
 274         TEST_ASSERT_NOT_NULL(mp,
 275                         "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
 276                         ops_pool_size,
 277                         ad->dev_id,
 278                         socket_id);
 279         ad->ops_mempool = mp;
 280
 281         /* Inputs */
 282         mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments);
 283         mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in");
 284         TEST_ASSERT_NOT_NULL(mp,
 285                         "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
 286                         mbuf_pool_size,
 287                         ad->dev_id,
 288                         socket_id);
 289         ad->in_mbuf_pool = mp;
 290
 291         /* Hard outputs */
 292         mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 293                         hard_out->nb_segments);
 294         mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size,
 295                         "hard_out");
 296         TEST_ASSERT_NOT_NULL(mp,
 297                         "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
 298                         mbuf_pool_size,
 299                         ad->dev_id,
 300                         socket_id);
 301         ad->hard_out_mbuf_pool = mp;
 302
 303         if (soft_out->nb_segments == 0)
 304                 return TEST_SUCCESS;
 305
 306         /* Soft outputs */
 307         mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 308                         soft_out->nb_segments);
 309         mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, mbuf_pool_size,
 310                         "soft_out");
 311         TEST_ASSERT_NOT_NULL(mp,
 312                         "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
 313                         mbuf_pool_size,
 314                         ad->dev_id,
 315                         socket_id);
 316         ad->soft_out_mbuf_pool = mp;
 317
 318         return 0;
 319 }
 320
 321 static int
 322 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
 323                 struct test_bbdev_vector *vector)
 324 {
 325         int ret;
 326         unsigned int queue_id;
 327         struct rte_bbdev_queue_conf qconf;
 328         struct active_device *ad = &active_devs[nb_active_devs];
 329         unsigned int nb_queues;
 330         enum rte_bbdev_op_type op_type = vector->op_type;
 331
 332         nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
 333         /* setup device */
 334         ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
 335         if (ret < 0) {
 336                 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
 337                                 dev_id, nb_queues, info->socket_id, ret);
 338                 return TEST_FAILED;
 339         }
 340
 341         /* configure interrupts if needed */
 342         if (intr_enabled) {
 343                 ret = rte_bbdev_intr_enable(dev_id);
 344                 if (ret < 0) {
 345                         printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
 346                                         ret);
 347                         return TEST_FAILED;
 348                 }
 349         }
 350
 351         /* setup device queues */
 352         qconf.socket = info->socket_id;
 353         qconf.queue_size = info->drv.default_queue_conf.queue_size;
 354         qconf.priority = 0;
 355         qconf.deferred_start = 0;
 356         qconf.op_type = op_type;
 357
 358         for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
 359                 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
 360                 if (ret != 0) {
 361                         printf(
 362                                         "Allocated all queues (id=%u) at prio%u on dev%u\n",
 363                                         queue_id, qconf.priority, dev_id);
 364                         qconf.priority++;
 365                         ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
 366                                         &qconf);
 367                 }
 368                 if (ret != 0) {
 369                         printf("All queues on dev %u allocated: %u\n",
 370                                         dev_id, queue_id);
 371                         break;
 372                 }
 373                 ad->queue_ids[queue_id] = queue_id;
 374         }
 375         TEST_ASSERT(queue_id != 0,
 376                         "ERROR Failed to configure any queues on dev %u",
 377                         dev_id);
 378         ad->nb_queues = queue_id;
 379
 380         set_avail_op(ad, op_type);
 381
 382         return TEST_SUCCESS;
 383 }
 384
 385 static int
 386 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
 387                 struct test_bbdev_vector *vector)
 388 {
 389         int ret;
 390
 391         active_devs[nb_active_devs].driver_name = info->drv.driver_name;
 392         active_devs[nb_active_devs].dev_id = dev_id;
 393
 394         ret = add_bbdev_dev(dev_id, info, vector);
 395         if (ret == TEST_SUCCESS)
 396                 ++nb_active_devs;
 397         return ret;
 398 }
 399
 400 static uint8_t
 401 populate_active_devices(void)
 402 {
 403         int ret;
 404         uint8_t dev_id;
 405         uint8_t nb_devs_added = 0;
 406         struct rte_bbdev_info info;
 407
 408         RTE_BBDEV_FOREACH(dev_id) {
 409                 rte_bbdev_info_get(dev_id, &info);
 410
 411                 if (check_dev_cap(&info)) {
 412                         printf(
 413                                 "Device %d (%s) does not support specified capabilities\n",
 414                                         dev_id, info.dev_name);
 415                         continue;
 416                 }
 417
 418                 ret = add_active_device(dev_id, &info, &test_vector);
 419                 if (ret != 0) {
 420                         printf("Adding active bbdev %s skipped\n",
 421                                         info.dev_name);
 422                         continue;
 423                 }
 424                 nb_devs_added++;
 425         }
 426
 427         return nb_devs_added;
 428 }
 429
 430 static int
 431 read_test_vector(void)
 432 {
 433         int ret;
 434
 435         memset(&test_vector, 0, sizeof(test_vector));
 436         printf("Test vector file = %s\n", get_vector_filename());
 437         ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
 438         TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
 439                         get_vector_filename());
 440
 441         return TEST_SUCCESS;
 442 }
 443
 444 static int
 445 testsuite_setup(void)
 446 {
 447         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 448
 449         if (populate_active_devices() == 0) {
 450                 printf("No suitable devices found!\n");
 451                 return TEST_SKIPPED;
 452         }
 453
 454         return TEST_SUCCESS;
 455 }
 456
 457 static int
 458 interrupt_testsuite_setup(void)
 459 {
 460         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 461
 462         /* Enable interrupts */
 463         intr_enabled = true;
 464
 465         /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
 466         if (populate_active_devices() == 0 ||
 467                         test_vector.op_type == RTE_BBDEV_OP_NONE) {
 468                 intr_enabled = false;
 469                 printf("No suitable devices found!\n");
 470                 return TEST_SKIPPED;
 471         }
 472
 473         return TEST_SUCCESS;
 474 }
 475
 476 static void
 477 testsuite_teardown(void)
 478 {
 479         uint8_t dev_id;
 480
 481         /* Unconfigure devices */
 482         RTE_BBDEV_FOREACH(dev_id)
 483                 rte_bbdev_close(dev_id);
 484
 485         /* Clear active devices structs. */
 486         memset(active_devs, 0, sizeof(active_devs));
 487         nb_active_devs = 0;
 488 }
 489
 490 static int
 491 ut_setup(void)
 492 {
 493         uint8_t i, dev_id;
 494
 495         for (i = 0; i < nb_active_devs; i++) {
 496                 dev_id = active_devs[i].dev_id;
 497                 /* reset bbdev stats */
 498                 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
 499                                 "Failed to reset stats of bbdev %u", dev_id);
 500                 /* start the device */
 501                 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
 502                                 "Failed to start bbdev %u", dev_id);
 503         }
 504
 505         return TEST_SUCCESS;
 506 }
 507
 508 static void
 509 ut_teardown(void)
 510 {
 511         uint8_t i, dev_id;
 512         struct rte_bbdev_stats stats;
 513
 514         for (i = 0; i < nb_active_devs; i++) {
 515                 dev_id = active_devs[i].dev_id;
 516                 /* read stats and print */
 517                 rte_bbdev_stats_get(dev_id, &stats);
 518                 /* Stop the device */
 519                 rte_bbdev_stop(dev_id);
 520         }
 521 }
 522
 523 static int
 524 init_op_data_objs(struct rte_bbdev_op_data *bufs,
 525                 struct op_data_entries *ref_entries,
 526                 struct rte_mempool *mbuf_pool, const uint16_t n,
 527                 enum op_data_type op_type, uint16_t min_alignment)
 528 {
 529         int ret;
 530         unsigned int i, j;
 531
 532         for (i = 0; i < n; ++i) {
 533                 char *data;
 534                 struct op_data_buf *seg = &ref_entries->segments[0];
 535                 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
 536                 TEST_ASSERT_NOT_NULL(m_head,
 537                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 538                                 op_type, n * ref_entries->nb_segments,
 539                                 mbuf_pool->size);
 540
 541                 bufs[i].data = m_head;
 542                 bufs[i].offset = 0;
 543                 bufs[i].length = 0;
 544
 545                 if (op_type == DATA_INPUT) {
 546                         data = rte_pktmbuf_append(m_head, seg->length);
 547                         TEST_ASSERT_NOT_NULL(data,
 548                                         "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 549                                         seg->length, op_type);
 550
 551                         TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment),
 552                                         "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 553                                         data, min_alignment);
 554                         rte_memcpy(data, seg->addr, seg->length);
 555                         bufs[i].length += seg->length;
 556
 557
 558                         for (j = 1; j < ref_entries->nb_segments; ++j) {
 559                                 struct rte_mbuf *m_tail =
 560                                                 rte_pktmbuf_alloc(mbuf_pool);
 561                                 TEST_ASSERT_NOT_NULL(m_tail,
 562                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 563                                                 op_type,
 564                                                 n * ref_entries->nb_segments,
 565                                                 mbuf_pool->size);
 566                                 seg += 1;
 567
 568                                 data = rte_pktmbuf_append(m_tail, seg->length);
 569                                 TEST_ASSERT_NOT_NULL(data,
 570                                                 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 571                                                 seg->length, op_type);
 572
 573                                 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
 574                                                 min_alignment),
 575                                                 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 576                                                 data, min_alignment);
 577                                 rte_memcpy(data, seg->addr, seg->length);
 578                                 bufs[i].length += seg->length;
 579
 580                                 ret = rte_pktmbuf_chain(m_head, m_tail);
 581                                 TEST_ASSERT_SUCCESS(ret,
 582                                                 "Couldn't chain mbufs from %d data type mbuf pool",
 583                                                 op_type);
 584                         }
 585                 }
 586         }
 587
 588         return 0;
 589 }
 590
 591 static int
 592 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
 593                 const int socket)
 594 {
 595         int i;
 596
 597         *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
 598         if (*buffers == NULL) {
 599                 printf("WARNING: Failed to allocate op_data on socket %d\n",
 600                                 socket);
 601                 /* try to allocate memory on other detected sockets */
 602                 for (i = 0; i < socket; i++) {
 603                         *buffers = rte_zmalloc_socket(NULL, len, 0, i);
 604                         if (*buffers != NULL)
 605                                 break;
 606                 }
 607         }
 608
 609         return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
 610 }
 611
 612 static int
 613 fill_queue_buffers(struct test_op_params *op_params,
 614                 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
 615                 struct rte_mempool *soft_out_mp, uint16_t queue_id,
 616                 uint16_t min_alignment, const int socket_id)
 617 {
 618         int ret;
 619         enum op_data_type type;
 620         const uint16_t n = op_params->num_to_process;
 621
 622         struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
 623                 in_mp,
 624                 soft_out_mp,
 625                 hard_out_mp,
 626         };
 627
 628         struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
 629                 &op_params->q_bufs[socket_id][queue_id].inputs,
 630                 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
 631                 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
 632         };
 633
 634         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
 635                 struct op_data_entries *ref_entries =
 636                                 &test_vector.entries[type];
 637                 if (ref_entries->nb_segments == 0)
 638                         continue;
 639
 640                 ret = allocate_buffers_on_socket(queue_ops[type],
 641                                 n * sizeof(struct rte_bbdev_op_data),
 642                                 socket_id);
 643                 TEST_ASSERT_SUCCESS(ret,
 644                                 "Couldn't allocate memory for rte_bbdev_op_data structs");
 645
 646                 ret = init_op_data_objs(*queue_ops[type], ref_entries,
 647                                 mbuf_pools[type], n, type, min_alignment);
 648                 TEST_ASSERT_SUCCESS(ret,
 649                                 "Couldn't init rte_bbdev_op_data structs");
 650         }
 651
 652         return 0;
 653 }
 654
 655 static void
 656 free_buffers(struct active_device *ad, struct test_op_params *op_params)
 657 {
 658         unsigned int i, j;
 659
 660         rte_mempool_free(ad->ops_mempool);
 661         rte_mempool_free(ad->in_mbuf_pool);
 662         rte_mempool_free(ad->hard_out_mbuf_pool);
 663         rte_mempool_free(ad->soft_out_mbuf_pool);
 664
 665         for (i = 0; i < rte_lcore_count(); ++i) {
 666                 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
 667                         rte_free(op_params->q_bufs[j][i].inputs);
 668                         rte_free(op_params->q_bufs[j][i].hard_outputs);
 669                         rte_free(op_params->q_bufs[j][i].soft_outputs);
 670                 }
 671         }
 672 }
 673
 674 static void
 675 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
 676                 unsigned int start_idx,
 677                 struct rte_bbdev_op_data *inputs,
 678                 struct rte_bbdev_op_data *hard_outputs,
 679                 struct rte_bbdev_op_data *soft_outputs,
 680                 struct rte_bbdev_dec_op *ref_op)
 681 {
 682         unsigned int i;
 683         struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
 684
 685         for (i = 0; i < n; ++i) {
 686                 if (turbo_dec->code_block_mode == 0) {
 687                         ops[i]->turbo_dec.tb_params.ea =
 688                                         turbo_dec->tb_params.ea;
 689                         ops[i]->turbo_dec.tb_params.eb =
 690                                         turbo_dec->tb_params.eb;
 691                         ops[i]->turbo_dec.tb_params.k_pos =
 692                                         turbo_dec->tb_params.k_pos;
 693                         ops[i]->turbo_dec.tb_params.k_neg =
 694                                         turbo_dec->tb_params.k_neg;
 695                         ops[i]->turbo_dec.tb_params.c =
 696                                         turbo_dec->tb_params.c;
 697                         ops[i]->turbo_dec.tb_params.c_neg =
 698                                         turbo_dec->tb_params.c_neg;
 699                         ops[i]->turbo_dec.tb_params.cab =
 700                                         turbo_dec->tb_params.cab;
 701                 } else {
 702                         ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
 703                         ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
 704                 }
 705
 706                 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
 707                 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
 708                 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
 709                 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
 710                 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
 711                 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
 712                 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
 713
 714                 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
 715                 ops[i]->turbo_dec.input = inputs[start_idx + i];
 716                 if (soft_outputs != NULL)
 717                         ops[i]->turbo_dec.soft_output =
 718                                 soft_outputs[start_idx + i];
 719         }
 720 }
 721
 722 static void
 723 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
 724                 unsigned int start_idx,
 725                 struct rte_bbdev_op_data *inputs,
 726                 struct rte_bbdev_op_data *outputs,
 727                 struct rte_bbdev_enc_op *ref_op)
 728 {
 729         unsigned int i;
 730         struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
 731         for (i = 0; i < n; ++i) {
 732                 if (turbo_enc->code_block_mode == 0) {
 733                         ops[i]->turbo_enc.tb_params.ea =
 734                                         turbo_enc->tb_params.ea;
 735                         ops[i]->turbo_enc.tb_params.eb =
 736                                         turbo_enc->tb_params.eb;
 737                         ops[i]->turbo_enc.tb_params.k_pos =
 738                                         turbo_enc->tb_params.k_pos;
 739                         ops[i]->turbo_enc.tb_params.k_neg =
 740                                         turbo_enc->tb_params.k_neg;
 741                         ops[i]->turbo_enc.tb_params.c =
 742                                         turbo_enc->tb_params.c;
 743                         ops[i]->turbo_enc.tb_params.c_neg =
 744                                         turbo_enc->tb_params.c_neg;
 745                         ops[i]->turbo_enc.tb_params.cab =
 746                                         turbo_enc->tb_params.cab;
 747                         ops[i]->turbo_enc.tb_params.ncb_pos =
 748                                         turbo_enc->tb_params.ncb_pos;
 749                         ops[i]->turbo_enc.tb_params.ncb_neg =
 750                                         turbo_enc->tb_params.ncb_neg;
 751                         ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
 752                 } else {
 753                         ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
 754                         ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
 755                         ops[i]->turbo_enc.cb_params.ncb =
 756                                         turbo_enc->cb_params.ncb;
 757                 }
 758                 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
 759                 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
 760                 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
 761
 762                 ops[i]->turbo_enc.output = outputs[start_idx + i];
 763                 ops[i]->turbo_enc.input = inputs[start_idx + i];
 764         }
 765 }
 766
 767 static int
 768 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
 769                 unsigned int order_idx, const int expected_status)
 770 {
 771         TEST_ASSERT(op->status == expected_status,
 772                         "op_status (%d) != expected_status (%d)",
 773                         op->status, expected_status);
 774
 775         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
 776                         "Ordering error, expected %p, got %p",
 777                         (void *)(uintptr_t)order_idx, op->opaque_data);
 778
 779         return TEST_SUCCESS;
 780 }
 781
 782 static int
 783 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
 784                 unsigned int order_idx, const int expected_status)
 785 {
 786         TEST_ASSERT(op->status == expected_status,
 787                         "op_status (%d) != expected_status (%d)",
 788                         op->status, expected_status);
 789
 790         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
 791                         "Ordering error, expected %p, got %p",
 792                         (void *)(uintptr_t)order_idx, op->opaque_data);
 793
 794         return TEST_SUCCESS;
 795 }
 796
 797 static inline int
 798 validate_op_chain(struct rte_bbdev_op_data *op,
 799                 struct op_data_entries *orig_op)
 800 {
 801         uint8_t i;
 802         struct rte_mbuf *m = op->data;
 803         uint8_t nb_dst_segments = orig_op->nb_segments;
 804
 805         TEST_ASSERT(nb_dst_segments == m->nb_segs,
 806                         "Number of segments differ in original (%u) and filled (%u) op",
 807                         nb_dst_segments, m->nb_segs);
 808
 809         for (i = 0; i < nb_dst_segments; ++i) {
 810                 /* Apply offset to the first mbuf segment */
 811                 uint16_t offset = (i == 0) ? op->offset : 0;
 812                 uint16_t data_len = m->data_len - offset;
 813
 814                 TEST_ASSERT(orig_op->segments[i].length == data_len,
 815                                 "Length of segment differ in original (%u) and filled (%u) op",
 816                                 orig_op->segments[i].length, data_len);
 817                 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
 818                                 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
 819                                 data_len,
 820                                 "Output buffers (CB=%u) are not equal", i);
 821                 m = m->next;
 822         }
 823
 824         return TEST_SUCCESS;
 825 }
 826
 827 static int
 828 validate_dec_buffers(struct rte_bbdev_dec_op *ref_op, struct test_buffers *bufs,
 829                 const uint16_t num_to_process)
 830 {
 831         int i;
 832
 833         struct op_data_entries *hard_data_orig =
 834                         &test_vector.entries[DATA_HARD_OUTPUT];
 835         struct op_data_entries *soft_data_orig =
 836                         &test_vector.entries[DATA_SOFT_OUTPUT];
 837
 838         for (i = 0; i < num_to_process; i++) {
 839                 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i],
 840                                 hard_data_orig),
 841                                 "Hard output buffers are not equal");
 842                 if (ref_op->turbo_dec.op_flags &
 843                                 RTE_BBDEV_TURBO_SOFT_OUTPUT)
 844                         TEST_ASSERT_SUCCESS(validate_op_chain(
 845                                         &bufs->soft_outputs[i],
 846                                         soft_data_orig),
 847                                         "Soft output buffers are not equal");
 848         }
 849
 850         return TEST_SUCCESS;
 851 }
 852
 853 static int
 854 validate_enc_buffers(struct test_buffers *bufs, const uint16_t num_to_process)
 855 {
 856         int i;
 857
 858         struct op_data_entries *hard_data_orig =
 859                         &test_vector.entries[DATA_HARD_OUTPUT];
 860
 861         for (i = 0; i < num_to_process; i++)
 862                 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i],
 863                                 hard_data_orig), "");
 864
 865         return TEST_SUCCESS;
 866 }
 867
 868 static int
 869 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
 870                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
 871 {
 872         unsigned int i;
 873         int ret;
 874         struct op_data_entries *hard_data_orig =
 875                         &test_vector.entries[DATA_HARD_OUTPUT];
 876         struct op_data_entries *soft_data_orig =
 877                         &test_vector.entries[DATA_SOFT_OUTPUT];
 878         struct rte_bbdev_op_turbo_dec *ops_td;
 879         struct rte_bbdev_op_data *hard_output;
 880         struct rte_bbdev_op_data *soft_output;
 881         struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
 882
 883         for (i = 0; i < n; ++i) {
 884                 ops_td = &ops[i]->turbo_dec;
 885                 hard_output = &ops_td->hard_output;
 886                 soft_output = &ops_td->soft_output;
 887
 888                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
 889                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
 890                                         "Returned iter_count (%d) > expected iter_count (%d)",
 891                                         ops_td->iter_count, ref_td->iter_count);
 892                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
 893                 TEST_ASSERT_SUCCESS(ret,
 894                                 "Checking status and ordering for decoder failed");
 895
 896                 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
 897                                 hard_data_orig),
 898                                 "Hard output buffers (CB=%u) are not equal",
 899                                 i);
 900
 901                 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
 902                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
 903                                         soft_data_orig),
 904                                         "Soft output buffers (CB=%u) are not equal",
 905                                         i);
 906         }
 907
 908         return TEST_SUCCESS;
 909 }
 910
 911 static int
 912 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
 913                 struct rte_bbdev_enc_op *ref_op)
 914 {
 915         unsigned int i;
 916         int ret;
 917         struct op_data_entries *hard_data_orig =
 918                         &test_vector.entries[DATA_HARD_OUTPUT];
 919
 920         for (i = 0; i < n; ++i) {
 921                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
 922                 TEST_ASSERT_SUCCESS(ret,
 923                                 "Checking status and ordering for encoder failed");
 924                 TEST_ASSERT_SUCCESS(validate_op_chain(
 925                                 &ops[i]->turbo_enc.output,
 926                                 hard_data_orig),
 927                                 "Output buffers (CB=%u) are not equal",
 928                                 i);
 929         }
 930
 931         return TEST_SUCCESS;
 932 }
 933
 934 static void
 935 create_reference_dec_op(struct rte_bbdev_dec_op *op)
 936 {
 937         unsigned int i;
 938         struct op_data_entries *entry;
 939
 940         op->turbo_dec = test_vector.turbo_dec;
 941         entry = &test_vector.entries[DATA_INPUT];
 942         for (i = 0; i < entry->nb_segments; ++i)
 943                 op->turbo_dec.input.length +=
 944                                 entry->segments[i].length;
 945 }
 946
 947 static void
 948 create_reference_enc_op(struct rte_bbdev_enc_op *op)
 949 {
 950         unsigned int i;
 951         struct op_data_entries *entry;
 952
 953         op->turbo_enc = test_vector.turbo_enc;
 954         entry = &test_vector.entries[DATA_INPUT];
 955         for (i = 0; i < entry->nb_segments; ++i)
 956                 op->turbo_enc.input.length +=
 957                                 entry->segments[i].length;
 958 }
 959
 960 static int
 961 init_test_op_params(struct test_op_params *op_params,
 962                 enum rte_bbdev_op_type op_type, const int expected_status,
 963                 const int vector_mask, struct rte_mempool *ops_mp,
 964                 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
 965 {
 966         int ret = 0;
 967         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
 968                 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
 969                                 &op_params->ref_dec_op, 1);
 970         else
 971                 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
 972                                 &op_params->ref_enc_op, 1);
 973
 974         TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
 975
 976         op_params->mp = ops_mp;
 977         op_params->burst_sz = burst_sz;
 978         op_params->num_to_process = num_to_process;
 979         op_params->num_lcores = num_lcores;
 980         op_params->vector_mask = vector_mask;
 981         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
 982                 op_params->ref_dec_op->status = expected_status;
 983         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
 984                 op_params->ref_enc_op->status = expected_status;
 985
 986         return 0;
 987 }
 988
 989 static int
 990 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
 991                 struct test_op_params *op_params)
 992 {
 993         int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
 994         unsigned int i;
 995         struct active_device *ad;
 996         unsigned int burst_sz = get_burst_sz();
 997         enum rte_bbdev_op_type op_type = test_vector.op_type;
 998
 999         ad = &active_devs[dev_id];
1000
1001         /* Check if device supports op_type */
1002         if (!is_avail_op(ad, test_vector.op_type))
1003                 return TEST_SUCCESS;
1004
1005         struct rte_bbdev_info info;
1006         rte_bbdev_info_get(ad->dev_id, &info);
1007         socket_id = GET_SOCKET(info.socket_id);
1008
1009         if (op_type == RTE_BBDEV_OP_NONE)
1010                 op_type = RTE_BBDEV_OP_TURBO_ENC;
1011         f_ret = create_mempools(ad, socket_id, op_type,
1012                         get_num_ops());
1013         if (f_ret != TEST_SUCCESS) {
1014                 printf("Couldn't create mempools");
1015                 goto fail;
1016         }
1017
1018         f_ret = init_test_op_params(op_params, test_vector.op_type,
1019                         test_vector.expected_status,
1020                         test_vector.mask,
1021                         ad->ops_mempool,
1022                         burst_sz,
1023                         get_num_ops(),
1024                         get_num_lcores());
1025         if (f_ret != TEST_SUCCESS) {
1026                 printf("Couldn't init test op params");
1027                 goto fail;
1028         }
1029
1030         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1031                 create_reference_dec_op(op_params->ref_dec_op);
1032         else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1033                 create_reference_enc_op(op_params->ref_enc_op);
1034
1035         for (i = 0; i < ad->nb_queues; ++i) {
1036                 f_ret = fill_queue_buffers(op_params,
1037                                 ad->in_mbuf_pool,
1038                                 ad->hard_out_mbuf_pool,
1039                                 ad->soft_out_mbuf_pool,
1040                                 ad->queue_ids[i],
1041                                 info.drv.min_alignment,
1042                                 socket_id);
1043                 if (f_ret != TEST_SUCCESS) {
1044                         printf("Couldn't init queue buffers");
1045                         goto fail;
1046                 }
1047         }
1048
1049         /* Run test case function */
1050         t_ret = test_case_func(ad, op_params);
1051
1052         /* Free active device resources and return */
1053         free_buffers(ad, op_params);
1054         return t_ret;
1055
1056 fail:
1057         free_buffers(ad, op_params);
1058         return TEST_FAILED;
1059 }
1060
1061 /* Run given test function per active device per supported op type
1062  * per burst size.
1063  */
1064 static int
1065 run_test_case(test_case_function *test_case_func)
1066 {
1067         int ret = 0;
1068         uint8_t dev;
1069
1070         /* Alloc op_params */
1071         struct test_op_params *op_params = rte_zmalloc(NULL,
1072                         sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
1073         TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
1074                         RTE_ALIGN(sizeof(struct test_op_params),
1075                                 RTE_CACHE_LINE_SIZE));
1076
1077         /* For each device run test case function */
1078         for (dev = 0; dev < nb_active_devs; ++dev)
1079                 ret |= run_test_case_on_device(test_case_func, dev, op_params);
1080
1081         rte_free(op_params);
1082
1083         return ret;
1084 }
1085
1086 static void
1087 dequeue_event_callback(uint16_t dev_id,
1088                 enum rte_bbdev_event_type event, void *cb_arg,
1089                 void *ret_param)
1090 {
1091         int ret;
1092         uint16_t i;
1093         uint64_t total_time;
1094         uint16_t deq, burst_sz, num_to_process;
1095         uint16_t queue_id = INVALID_QUEUE_ID;
1096         struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
1097         struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
1098         struct test_buffers *bufs;
1099         struct rte_bbdev_info info;
1100
1101         /* Input length in bytes, million operations per second,
1102          * million bits per second.
1103          */
1104         double in_len;
1105
1106         struct thread_params *tp = cb_arg;
1107
1108         RTE_SET_USED(ret_param);
1109         queue_id = tp->queue_id;
1110
1111         /* Find matching thread params using queue_id */
1112         for (i = 0; i < MAX_QUEUES; ++i, ++tp)
1113                 if (tp->queue_id == queue_id)
1114                         break;
1115
1116         if (i == MAX_QUEUES) {
1117                 printf("%s: Queue_id from interrupt details was not found!\n",
1118                                 __func__);
1119                 return;
1120         }
1121
1122         if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
1123                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1124                 printf(
1125                         "Dequeue interrupt handler called for incorrect event!\n");
1126                 return;
1127         }
1128
1129         burst_sz = tp->op_params->burst_sz;
1130         num_to_process = tp->op_params->num_to_process;
1131
1132         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1133                 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, dec_ops,
1134                                 burst_sz);
1135         else
1136                 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, enc_ops,
1137                                 burst_sz);
1138
1139         if (deq < burst_sz) {
1140                 printf(
1141                         "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
1142                         burst_sz, deq);
1143                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1144                 return;
1145         }
1146
1147         if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_to_process) {
1148                 rte_atomic16_add(&tp->nb_dequeued, deq);
1149                 return;
1150         }
1151
1152         total_time = rte_rdtsc_precise() - tp->start_time;
1153
1154         rte_bbdev_info_get(dev_id, &info);
1155
1156         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1157
1158         ret = TEST_SUCCESS;
1159         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1160                 ret = validate_dec_buffers(tp->op_params->ref_dec_op, bufs,
1161                                 num_to_process);
1162         else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1163                 ret = validate_enc_buffers(bufs, num_to_process);
1164
1165         if (ret) {
1166                 printf("Buffers validation failed\n");
1167                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1168         }
1169
1170         switch (test_vector.op_type) {
1171         case RTE_BBDEV_OP_TURBO_DEC:
1172                 in_len = tp->op_params->ref_dec_op->turbo_dec.input.length;
1173                 break;
1174         case RTE_BBDEV_OP_TURBO_ENC:
1175                 in_len = tp->op_params->ref_enc_op->turbo_enc.input.length;
1176                 break;
1177         case RTE_BBDEV_OP_NONE:
1178                 in_len = 0.0;
1179                 break;
1180         default:
1181                 printf("Unknown op type: %d\n", test_vector.op_type);
1182                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1183                 return;
1184         }
1185
1186         tp->mops = ((double)num_to_process / 1000000.0) /
1187                         ((double)total_time / (double)rte_get_tsc_hz());
1188         tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1189                         ((double)total_time / (double)rte_get_tsc_hz());
1190
1191         rte_atomic16_add(&tp->nb_dequeued, deq);
1192 }
1193
1194 static int
1195 throughput_intr_lcore_dec(void *arg)
1196 {
1197         struct thread_params *tp = arg;
1198         unsigned int enqueued;
1199         struct rte_bbdev_dec_op *ops[MAX_BURST];
1200         const uint16_t queue_id = tp->queue_id;
1201         const uint16_t burst_sz = tp->op_params->burst_sz;
1202         const uint16_t num_to_process = tp->op_params->num_to_process;
1203         struct test_buffers *bufs = NULL;
1204         unsigned int allocs_failed = 0;
1205         struct rte_bbdev_info info;
1206         int ret;
1207
1208         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1209                         "BURST_SIZE should be <= %u", MAX_BURST);
1210
1211         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1212                         "Failed to enable interrupts for dev: %u, queue_id: %u",
1213                         tp->dev_id, queue_id);
1214
1215         rte_bbdev_info_get(tp->dev_id, &info);
1216         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1217
1218         rte_atomic16_clear(&tp->processing_status);
1219         rte_atomic16_clear(&tp->nb_dequeued);
1220
1221         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1222                 rte_pause();
1223
1224         tp->start_time = rte_rdtsc_precise();
1225         for (enqueued = 0; enqueued < num_to_process;) {
1226
1227                 uint16_t num_to_enq = burst_sz;
1228
1229                 if (unlikely(num_to_process - enqueued < num_to_enq))
1230                         num_to_enq = num_to_process - enqueued;
1231
1232                 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
1233                                 num_to_enq);
1234                 if (ret != 0) {
1235                         allocs_failed++;
1236                         continue;
1237                 }
1238
1239                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1240                         copy_reference_dec_op(ops, num_to_enq, enqueued,
1241                                         bufs->inputs,
1242                                         bufs->hard_outputs,
1243                                         bufs->soft_outputs,
1244                                         tp->op_params->ref_dec_op);
1245
1246                 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, queue_id, ops,
1247                                 num_to_enq);
1248
1249                 rte_bbdev_dec_op_free_bulk(ops, num_to_enq);
1250         }
1251
1252         if (allocs_failed > 0)
1253                 printf("WARNING: op allocations failed: %u times\n",
1254                                 allocs_failed);
1255
1256         return TEST_SUCCESS;
1257 }
1258
1259 static int
1260 throughput_intr_lcore_enc(void *arg)
1261 {
1262         struct thread_params *tp = arg;
1263         unsigned int enqueued;
1264         struct rte_bbdev_enc_op *ops[MAX_BURST];
1265         const uint16_t queue_id = tp->queue_id;
1266         const uint16_t burst_sz = tp->op_params->burst_sz;
1267         const uint16_t num_to_process = tp->op_params->num_to_process;
1268         struct test_buffers *bufs = NULL;
1269         unsigned int allocs_failed = 0;
1270         struct rte_bbdev_info info;
1271         int ret;
1272
1273         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1274                         "BURST_SIZE should be <= %u", MAX_BURST);
1275
1276         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1277                         "Failed to enable interrupts for dev: %u, queue_id: %u",
1278                         tp->dev_id, queue_id);
1279
1280         rte_bbdev_info_get(tp->dev_id, &info);
1281         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1282
1283         rte_atomic16_clear(&tp->processing_status);
1284         rte_atomic16_clear(&tp->nb_dequeued);
1285
1286         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1287                 rte_pause();
1288
1289         tp->start_time = rte_rdtsc_precise();
1290         for (enqueued = 0; enqueued < num_to_process;) {
1291
1292                 uint16_t num_to_enq = burst_sz;
1293
1294                 if (unlikely(num_to_process - enqueued < num_to_enq))
1295                         num_to_enq = num_to_process - enqueued;
1296
1297                 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
1298                                 num_to_enq);
1299                 if (ret != 0) {
1300                         allocs_failed++;
1301                         continue;
1302                 }
1303
1304                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1305                         copy_reference_enc_op(ops, num_to_enq, enqueued,
1306                                         bufs->inputs,
1307                                         bufs->hard_outputs,
1308                                         tp->op_params->ref_enc_op);
1309
1310                 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, queue_id, ops,
1311                                 num_to_enq);
1312
1313                 rte_bbdev_enc_op_free_bulk(ops, num_to_enq);
1314         }
1315
1316         if (allocs_failed > 0)
1317                 printf("WARNING: op allocations failed: %u times\n",
1318                                 allocs_failed);
1319
1320         return TEST_SUCCESS;
1321 }
1322
1323 static int
1324 throughput_pmd_lcore_dec(void *arg)
1325 {
1326         struct thread_params *tp = arg;
1327         unsigned int enqueued, dequeued;
1328         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1329         uint64_t total_time, start_time;
1330         const uint16_t queue_id = tp->queue_id;
1331         const uint16_t burst_sz = tp->op_params->burst_sz;
1332         const uint16_t num_to_process = tp->op_params->num_to_process;
1333         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1334         struct test_buffers *bufs = NULL;
1335         unsigned int allocs_failed = 0;
1336         int ret;
1337         struct rte_bbdev_info info;
1338
1339         /* Input length in bytes, million operations per second, million bits
1340          * per second.
1341          */
1342         double in_len;
1343
1344         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1345                         "BURST_SIZE should be <= %u", MAX_BURST);
1346
1347         rte_bbdev_info_get(tp->dev_id, &info);
1348         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1349
1350         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1351                 rte_pause();
1352
1353         start_time = rte_rdtsc_precise();
1354         for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) {
1355                 uint16_t deq;
1356
1357                 if (likely(enqueued < num_to_process)) {
1358
1359                         uint16_t num_to_enq = burst_sz;
1360
1361                         if (unlikely(num_to_process - enqueued < num_to_enq))
1362                                 num_to_enq = num_to_process - enqueued;
1363
1364                         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp,
1365                                         ops_enq, num_to_enq);
1366                         if (ret != 0) {
1367                                 allocs_failed++;
1368                                 goto do_dequeue;
1369                         }
1370
1371                         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1372                                 copy_reference_dec_op(ops_enq, num_to_enq,
1373                                                 enqueued,
1374                                                 bufs->inputs,
1375                                                 bufs->hard_outputs,
1376                                                 bufs->soft_outputs,
1377                                                 ref_op);
1378
1379                         enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id,
1380                                         queue_id, ops_enq, num_to_enq);
1381                 }
1382 do_dequeue:
1383                 deq = rte_bbdev_dequeue_dec_ops(tp->dev_id, queue_id, ops_deq,
1384                                 burst_sz);
1385                 dequeued += deq;
1386                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1387         }
1388         total_time = rte_rdtsc_precise() - start_time;
1389
1390         if (allocs_failed > 0)
1391                 printf("WARNING: op allocations failed: %u times\n",
1392                                 allocs_failed);
1393
1394         TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)",
1395                         enqueued, dequeued);
1396
1397         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1398                 ret = validate_dec_buffers(ref_op, bufs, num_to_process);
1399                 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed");
1400         }
1401
1402         in_len = ref_op->turbo_dec.input.length;
1403         tp->mops = ((double)num_to_process / 1000000.0) /
1404                         ((double)total_time / (double)rte_get_tsc_hz());
1405         tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1406                         ((double)total_time / (double)rte_get_tsc_hz());
1407
1408         return TEST_SUCCESS;
1409 }
1410
1411 static int
1412 throughput_pmd_lcore_enc(void *arg)
1413 {
1414         struct thread_params *tp = arg;
1415         unsigned int enqueued, dequeued;
1416         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1417         uint64_t total_time, start_time;
1418         const uint16_t queue_id = tp->queue_id;
1419         const uint16_t burst_sz = tp->op_params->burst_sz;
1420         const uint16_t num_to_process = tp->op_params->num_to_process;
1421         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1422         struct test_buffers *bufs = NULL;
1423         unsigned int allocs_failed = 0;
1424         int ret;
1425         struct rte_bbdev_info info;
1426
1427         /* Input length in bytes, million operations per second, million bits
1428          * per second.
1429          */
1430         double in_len;
1431
1432         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1433                         "BURST_SIZE should be <= %u", MAX_BURST);
1434
1435         rte_bbdev_info_get(tp->dev_id, &info);
1436         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1437
1438         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1439                 rte_pause();
1440
1441         start_time = rte_rdtsc_precise();
1442         for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) {
1443                 uint16_t deq;
1444
1445                 if (likely(enqueued < num_to_process)) {
1446
1447                         uint16_t num_to_enq = burst_sz;
1448
1449                         if (unlikely(num_to_process - enqueued < num_to_enq))
1450                                 num_to_enq = num_to_process - enqueued;
1451
1452                         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp,
1453                                         ops_enq, num_to_enq);
1454                         if (ret != 0) {
1455                                 allocs_failed++;
1456                                 goto do_dequeue;
1457                         }
1458
1459                         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1460                                 copy_reference_enc_op(ops_enq, num_to_enq,
1461                                                 enqueued,
1462                                                 bufs->inputs,
1463                                                 bufs->hard_outputs,
1464                                                 ref_op);
1465
1466                         enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id,
1467                                         queue_id, ops_enq, num_to_enq);
1468                 }
1469 do_dequeue:
1470                 deq = rte_bbdev_dequeue_enc_ops(tp->dev_id, queue_id, ops_deq,
1471                                 burst_sz);
1472                 dequeued += deq;
1473                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1474         }
1475         total_time = rte_rdtsc_precise() - start_time;
1476
1477         if (allocs_failed > 0)
1478                 printf("WARNING: op allocations failed: %u times\n",
1479                                 allocs_failed);
1480
1481         TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)",
1482                         enqueued, dequeued);
1483
1484         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1485                 ret = validate_enc_buffers(bufs, num_to_process);
1486                 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed");
1487         }
1488
1489         in_len = ref_op->turbo_enc.input.length;
1490
1491         tp->mops = ((double)num_to_process / 1000000.0) /
1492                         ((double)total_time / (double)rte_get_tsc_hz());
1493         tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1494                         ((double)total_time / (double)rte_get_tsc_hz());
1495
1496         return TEST_SUCCESS;
1497 }
1498 static void
1499 print_throughput(struct thread_params *t_params, unsigned int used_cores)
1500 {
1501         unsigned int lcore_id, iter = 0;
1502         double total_mops = 0, total_mbps = 0;
1503
1504         RTE_LCORE_FOREACH(lcore_id) {
1505                 if (iter++ >= used_cores)
1506                         break;
1507                 printf("\tlcore_id: %u, throughput: %.8lg MOPS, %.8lg Mbps\n",
1508                 lcore_id, t_params[lcore_id].mops, t_params[lcore_id].mbps);
1509                 total_mops += t_params[lcore_id].mops;
1510                 total_mbps += t_params[lcore_id].mbps;
1511         }
1512         printf(
1513                 "\n\tTotal stats for %u cores: throughput: %.8lg MOPS, %.8lg Mbps\n",
1514                 used_cores, total_mops, total_mbps);
1515 }
1516
1517 /*
1518  * Test function that determines how long an enqueue + dequeue of a burst
1519  * takes on available lcores.
1520  */
1521 static int
1522 throughput_test(struct active_device *ad,
1523                 struct test_op_params *op_params)
1524 {
1525         int ret;
1526         unsigned int lcore_id, used_cores = 0;
1527         struct thread_params t_params[MAX_QUEUES];
1528         struct rte_bbdev_info info;
1529         lcore_function_t *throughput_function;
1530         struct thread_params *tp;
1531         uint16_t num_lcores;
1532         const char *op_type_str;
1533
1534         rte_bbdev_info_get(ad->dev_id, &info);
1535
1536         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
1537         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
1538                         test_vector.op_type);
1539
1540         printf(
1541                 "Throughput test: dev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, int mode: %s, GHz: %lg\n",
1542                         info.dev_name, ad->nb_queues, op_params->burst_sz,
1543                         op_params->num_to_process, op_params->num_lcores,
1544                         op_type_str,
1545                         intr_enabled ? "Interrupt mode" : "PMD mode",
1546                         (double)rte_get_tsc_hz() / 1000000000.0);
1547
1548         /* Set number of lcores */
1549         num_lcores = (ad->nb_queues < (op_params->num_lcores))
1550                         ? ad->nb_queues
1551                         : op_params->num_lcores;
1552
1553         if (intr_enabled) {
1554                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1555                         throughput_function = throughput_intr_lcore_dec;
1556                 else
1557                         throughput_function = throughput_intr_lcore_enc;
1558
1559                 /* Dequeue interrupt callback registration */
1560                 rte_bbdev_callback_register(ad->dev_id, RTE_BBDEV_EVENT_DEQUEUE,
1561                                 dequeue_event_callback,
1562                                 &t_params);
1563         } else {
1564                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1565                         throughput_function = throughput_pmd_lcore_dec;
1566                 else
1567                         throughput_function = throughput_pmd_lcore_enc;
1568         }
1569
1570         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
1571
1572         t_params[rte_lcore_id()].dev_id = ad->dev_id;
1573         t_params[rte_lcore_id()].op_params = op_params;
1574         t_params[rte_lcore_id()].queue_id =
1575                         ad->queue_ids[used_cores++];
1576
1577         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1578                 if (used_cores >= num_lcores)
1579                         break;
1580
1581                 t_params[lcore_id].dev_id = ad->dev_id;
1582                 t_params[lcore_id].op_params = op_params;
1583                 t_params[lcore_id].queue_id = ad->queue_ids[used_cores++];
1584
1585                 rte_eal_remote_launch(throughput_function, &t_params[lcore_id],
1586                                 lcore_id);
1587         }
1588
1589         rte_atomic16_set(&op_params->sync, SYNC_START);
1590         ret = throughput_function(&t_params[rte_lcore_id()]);
1591
1592         /* Master core is always used */
1593         used_cores = 1;
1594         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1595                 if (used_cores++ >= num_lcores)
1596                         break;
1597
1598                 ret |= rte_eal_wait_lcore(lcore_id);
1599         }
1600
1601         /* Return if test failed */
1602         if (ret)
1603                 return ret;
1604
1605         /* Print throughput if interrupts are disabled and test passed */
1606         if (!intr_enabled) {
1607                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1608                         print_throughput(t_params, num_lcores);
1609                 return ret;
1610         }
1611
1612         /* In interrupt TC we need to wait for the interrupt callback to deqeue
1613          * all pending operations. Skip waiting for queues which reported an
1614          * error using processing_status variable.
1615          * Wait for master lcore operations.
1616          */
1617         tp = &t_params[rte_lcore_id()];
1618         while ((rte_atomic16_read(&tp->nb_dequeued) <
1619                         op_params->num_to_process) &&
1620                         (rte_atomic16_read(&tp->processing_status) !=
1621                         TEST_FAILED))
1622                 rte_pause();
1623
1624         ret |= rte_atomic16_read(&tp->processing_status);
1625
1626         /* Wait for slave lcores operations */
1627         used_cores = 1;
1628         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1629                 tp = &t_params[lcore_id];
1630                 if (used_cores++ >= num_lcores)
1631                         break;
1632
1633                 while ((rte_atomic16_read(&tp->nb_dequeued) <
1634                                 op_params->num_to_process) &&
1635                                 (rte_atomic16_read(&tp->processing_status) !=
1636                                 TEST_FAILED))
1637                         rte_pause();
1638
1639                 ret |= rte_atomic16_read(&tp->processing_status);
1640         }
1641
1642         /* Print throughput if test passed */
1643         if (!ret && test_vector.op_type != RTE_BBDEV_OP_NONE)
1644                 print_throughput(t_params, num_lcores);
1645
1646         return ret;
1647 }
1648
1649 static int
1650 operation_latency_test_dec(struct rte_mempool *mempool,
1651                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
1652                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
1653                 const uint16_t num_to_process, uint16_t burst_sz,
1654                 uint64_t *total_time)
1655 {
1656         int ret = TEST_SUCCESS;
1657         uint16_t i, j, dequeued;
1658         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1659         uint64_t start_time = 0;
1660
1661         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1662                 uint16_t enq = 0, deq = 0;
1663                 bool first_time = true;
1664
1665                 if (unlikely(num_to_process - dequeued < burst_sz))
1666                         burst_sz = num_to_process - dequeued;
1667
1668                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1669                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1670                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1671                                         bufs->inputs,
1672                                         bufs->hard_outputs,
1673                                         bufs->soft_outputs,
1674                                         ref_op);
1675
1676                 /* Set counter to validate the ordering */
1677                 for (j = 0; j < burst_sz; ++j)
1678                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1679
1680                 start_time = rte_rdtsc_precise();
1681
1682                 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
1683                                 burst_sz);
1684                 TEST_ASSERT(enq == burst_sz,
1685                                 "Error enqueueing burst, expected %u, got %u",
1686                                 burst_sz, enq);
1687
1688                 /* Dequeue */
1689                 do {
1690                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1691                                         &ops_deq[deq], burst_sz - deq);
1692                         if (likely(first_time && (deq > 0))) {
1693                                 *total_time += rte_rdtsc_precise() - start_time;
1694                                 first_time = false;
1695                         }
1696                 } while (unlikely(burst_sz != deq));
1697
1698                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1699                         ret = validate_dec_op(ops_deq, burst_sz, ref_op,
1700                                         vector_mask);
1701                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1702                 }
1703
1704                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1705                 dequeued += deq;
1706         }
1707
1708         return i;
1709 }
1710
1711 static int
1712 operation_latency_test_enc(struct rte_mempool *mempool,
1713                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
1714                 uint16_t dev_id, uint16_t queue_id,
1715                 const uint16_t num_to_process, uint16_t burst_sz,
1716                 uint64_t *total_time)
1717 {
1718         int ret = TEST_SUCCESS;
1719         uint16_t i, j, dequeued;
1720         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1721         uint64_t start_time = 0;
1722
1723         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1724                 uint16_t enq = 0, deq = 0;
1725                 bool first_time = true;
1726
1727                 if (unlikely(num_to_process - dequeued < burst_sz))
1728                         burst_sz = num_to_process - dequeued;
1729
1730                 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
1731                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1732                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
1733                                         bufs->inputs,
1734                                         bufs->hard_outputs,
1735                                         ref_op);
1736
1737                 /* Set counter to validate the ordering */
1738                 for (j = 0; j < burst_sz; ++j)
1739                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1740
1741                 start_time = rte_rdtsc_precise();
1742
1743                 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
1744                                 burst_sz);
1745                 TEST_ASSERT(enq == burst_sz,
1746                                 "Error enqueueing burst, expected %u, got %u",
1747                                 burst_sz, enq);
1748
1749                 /* Dequeue */
1750                 do {
1751                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1752                                         &ops_deq[deq], burst_sz - deq);
1753                         if (likely(first_time && (deq > 0))) {
1754                                 *total_time += rte_rdtsc_precise() - start_time;
1755                                 first_time = false;
1756                         }
1757                 } while (unlikely(burst_sz != deq));
1758
1759                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1760                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
1761                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1762                 }
1763
1764                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1765                 dequeued += deq;
1766         }
1767
1768         return i;
1769 }
1770
1771 static int
1772 operation_latency_test(struct active_device *ad,
1773                 struct test_op_params *op_params)
1774 {
1775         int iter;
1776         uint16_t burst_sz = op_params->burst_sz;
1777         const uint16_t num_to_process = op_params->num_to_process;
1778         const enum rte_bbdev_op_type op_type = test_vector.op_type;
1779         const uint16_t queue_id = ad->queue_ids[0];
1780         struct test_buffers *bufs = NULL;
1781         struct rte_bbdev_info info;
1782         uint64_t total_time = 0;
1783         const char *op_type_str;
1784
1785         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1786                         "BURST_SIZE should be <= %u", MAX_BURST);
1787
1788         rte_bbdev_info_get(ad->dev_id, &info);
1789         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1790
1791         op_type_str = rte_bbdev_op_type_str(op_type);
1792         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
1793
1794         printf(
1795                 "Validation/Latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
1796                         info.dev_name, burst_sz, num_to_process, op_type_str);
1797
1798         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1799                 iter = operation_latency_test_dec(op_params->mp, bufs,
1800                                 op_params->ref_dec_op, op_params->vector_mask,
1801                                 ad->dev_id, queue_id, num_to_process,
1802                                 burst_sz, &total_time);
1803         else
1804                 iter = operation_latency_test_enc(op_params->mp, bufs,
1805                                 op_params->ref_enc_op, ad->dev_id, queue_id,
1806                                 num_to_process, burst_sz, &total_time);
1807
1808         if (iter <= 0)
1809                 return TEST_FAILED;
1810
1811         printf("\toperation avg. latency: %lg cycles, %lg us\n",
1812                         (double)total_time / (double)iter,
1813                         (double)(total_time * 1000000) / (double)iter /
1814                         (double)rte_get_tsc_hz());
1815
1816         return TEST_SUCCESS;
1817 }
1818
1819 static int
1820 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
1821                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
1822                 uint16_t queue_id, const uint16_t num_to_process,
1823                 uint16_t burst_sz, uint64_t *enq_total_time,
1824                 uint64_t *deq_total_time)
1825 {
1826         int i, dequeued;
1827         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1828         uint64_t enq_start_time, deq_start_time;
1829
1830         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1831                 uint16_t enq = 0, deq = 0;
1832
1833                 if (unlikely(num_to_process - dequeued < burst_sz))
1834                         burst_sz = num_to_process - dequeued;
1835
1836                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1837                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1838                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1839                                         bufs->inputs,
1840                                         bufs->hard_outputs,
1841                                         bufs->soft_outputs,
1842                                         ref_op);
1843
1844                 /* Start time measurment for enqueue function offload latency */
1845                 enq_start_time = rte_rdtsc();
1846                 do {
1847                         enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
1848                                         &ops_enq[enq], burst_sz - enq);
1849                 } while (unlikely(burst_sz != enq));
1850                 *enq_total_time += rte_rdtsc() - enq_start_time;
1851
1852                 /* ensure enqueue has been completed */
1853                 rte_delay_ms(10);
1854
1855                 /* Start time measurment for dequeue function offload latency */
1856                 deq_start_time = rte_rdtsc();
1857                 do {
1858                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1859                                         &ops_deq[deq], burst_sz - deq);
1860                 } while (unlikely(burst_sz != deq));
1861                 *deq_total_time += rte_rdtsc() - deq_start_time;
1862
1863                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1864                 dequeued += deq;
1865         }
1866
1867         return i;
1868 }
1869
1870 static int
1871 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
1872                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
1873                 uint16_t queue_id, const uint16_t num_to_process,
1874                 uint16_t burst_sz, uint64_t *enq_total_time,
1875                 uint64_t *deq_total_time)
1876 {
1877         int i, dequeued;
1878         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1879         uint64_t enq_start_time, deq_start_time;
1880
1881         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1882                 uint16_t enq = 0, deq = 0;
1883
1884                 if (unlikely(num_to_process - dequeued < burst_sz))
1885                         burst_sz = num_to_process - dequeued;
1886
1887                 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
1888                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1889                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
1890                                         bufs->inputs,
1891                                         bufs->hard_outputs,
1892                                         ref_op);
1893
1894                 /* Start time measurment for enqueue function offload latency */
1895                 enq_start_time = rte_rdtsc();
1896                 do {
1897                         enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
1898                                         &ops_enq[enq], burst_sz - enq);
1899                 } while (unlikely(burst_sz != enq));
1900                 *enq_total_time += rte_rdtsc() - enq_start_time;
1901
1902                 /* ensure enqueue has been completed */
1903                 rte_delay_ms(10);
1904
1905                 /* Start time measurment for dequeue function offload latency */
1906                 deq_start_time = rte_rdtsc();
1907                 do {
1908                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1909                                         &ops_deq[deq], burst_sz - deq);
1910                 } while (unlikely(burst_sz != deq));
1911                 *deq_total_time += rte_rdtsc() - deq_start_time;
1912
1913                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1914                 dequeued += deq;
1915         }
1916
1917         return i;
1918 }
1919
1920 static int
1921 offload_latency_test(struct active_device *ad,
1922                 struct test_op_params *op_params)
1923 {
1924         int iter;
1925         uint64_t enq_total_time = 0, deq_total_time = 0;
1926         uint16_t burst_sz = op_params->burst_sz;
1927         const uint16_t num_to_process = op_params->num_to_process;
1928         const enum rte_bbdev_op_type op_type = test_vector.op_type;
1929         const uint16_t queue_id = ad->queue_ids[0];
1930         struct test_buffers *bufs = NULL;
1931         struct rte_bbdev_info info;
1932         const char *op_type_str;
1933
1934         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1935                         "BURST_SIZE should be <= %u", MAX_BURST);
1936
1937         rte_bbdev_info_get(ad->dev_id, &info);
1938         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1939
1940         op_type_str = rte_bbdev_op_type_str(op_type);
1941         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
1942
1943         printf(
1944                 "Offload latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
1945                         info.dev_name, burst_sz, num_to_process, op_type_str);
1946
1947         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1948                 iter = offload_latency_test_dec(op_params->mp, bufs,
1949                                 op_params->ref_dec_op, ad->dev_id, queue_id,
1950                                 num_to_process, burst_sz, &enq_total_time,
1951                                 &deq_total_time);
1952         else
1953                 iter = offload_latency_test_enc(op_params->mp, bufs,
1954                                 op_params->ref_enc_op, ad->dev_id, queue_id,
1955                                 num_to_process, burst_sz, &enq_total_time,
1956                                 &deq_total_time);
1957
1958         if (iter <= 0)
1959                 return TEST_FAILED;
1960
1961         printf("\tenq offload avg. latency: %lg cycles, %lg us\n",
1962                         (double)enq_total_time / (double)iter,
1963                         (double)(enq_total_time * 1000000) / (double)iter /
1964                         (double)rte_get_tsc_hz());
1965
1966         printf("\tdeq offload avg. latency: %lg cycles, %lg us\n",
1967                         (double)deq_total_time / (double)iter,
1968                         (double)(deq_total_time * 1000000) / (double)iter /
1969                         (double)rte_get_tsc_hz());
1970
1971         return TEST_SUCCESS;
1972 }
1973
1974 static int
1975 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
1976                 const uint16_t num_to_process, uint16_t burst_sz,
1977                 uint64_t *deq_total_time)
1978 {
1979         int i, deq_total;
1980         struct rte_bbdev_dec_op *ops[MAX_BURST];
1981         uint64_t deq_start_time;
1982
1983         /* Test deq offload latency from an empty queue */
1984         deq_start_time = rte_rdtsc_precise();
1985         for (i = 0, deq_total = 0; deq_total < num_to_process;
1986                         ++i, deq_total += burst_sz) {
1987                 if (unlikely(num_to_process - deq_total < burst_sz))
1988                         burst_sz = num_to_process - deq_total;
1989                 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz);
1990         }
1991         *deq_total_time = rte_rdtsc_precise() - deq_start_time;
1992
1993         return i;
1994 }
1995
1996 static int
1997 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
1998                 const uint16_t num_to_process, uint16_t burst_sz,
1999                 uint64_t *deq_total_time)
2000 {
2001         int i, deq_total;
2002         struct rte_bbdev_enc_op *ops[MAX_BURST];
2003         uint64_t deq_start_time;
2004
2005         /* Test deq offload latency from an empty queue */
2006         deq_start_time = rte_rdtsc_precise();
2007         for (i = 0, deq_total = 0; deq_total < num_to_process;
2008                         ++i, deq_total += burst_sz) {
2009                 if (unlikely(num_to_process - deq_total < burst_sz))
2010                         burst_sz = num_to_process - deq_total;
2011                 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz);
2012         }
2013         *deq_total_time = rte_rdtsc_precise() - deq_start_time;
2014
2015         return i;
2016 }
2017
2018 static int
2019 offload_latency_empty_q_test(struct active_device *ad,
2020                 struct test_op_params *op_params)
2021 {
2022         int iter;
2023         uint64_t deq_total_time = 0;
2024         uint16_t burst_sz = op_params->burst_sz;
2025         const uint16_t num_to_process = op_params->num_to_process;
2026         const enum rte_bbdev_op_type op_type = test_vector.op_type;
2027         const uint16_t queue_id = ad->queue_ids[0];
2028         struct rte_bbdev_info info;
2029         const char *op_type_str;
2030
2031         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2032                         "BURST_SIZE should be <= %u", MAX_BURST);
2033
2034         rte_bbdev_info_get(ad->dev_id, &info);
2035
2036         op_type_str = rte_bbdev_op_type_str(op_type);
2037         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2038
2039         printf(
2040                 "Offload latency empty dequeue test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
2041                         info.dev_name, burst_sz, num_to_process, op_type_str);
2042
2043         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2044                 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
2045                                 num_to_process, burst_sz, &deq_total_time);
2046         else
2047                 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
2048                                 num_to_process, burst_sz, &deq_total_time);
2049
2050         if (iter <= 0)
2051                 return TEST_FAILED;
2052
2053         printf("\tempty deq offload avg. latency: %lg cycles, %lg us\n",
2054                         (double)deq_total_time / (double)iter,
2055                         (double)(deq_total_time * 1000000) / (double)iter /
2056                         (double)rte_get_tsc_hz());
2057
2058         return TEST_SUCCESS;
2059 }
2060
2061 static int
2062 throughput_tc(void)
2063 {
2064         return run_test_case(throughput_test);
2065 }
2066
2067 static int
2068 offload_latency_tc(void)
2069 {
2070         return run_test_case(offload_latency_test);
2071 }
2072
2073 static int
2074 offload_latency_empty_q_tc(void)
2075 {
2076         return run_test_case(offload_latency_empty_q_test);
2077 }
2078
2079 static int
2080 operation_latency_tc(void)
2081 {
2082         return run_test_case(operation_latency_test);
2083 }
2084
2085 static int
2086 interrupt_tc(void)
2087 {
2088         return run_test_case(throughput_test);
2089 }
2090
2091 static struct unit_test_suite bbdev_throughput_testsuite = {
2092         .suite_name = "BBdev Throughput Tests",
2093         .setup = testsuite_setup,
2094         .teardown = testsuite_teardown,
2095         .unit_test_cases = {
2096                 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
2097                 TEST_CASES_END() /**< NULL terminate unit test array */
2098         }
2099 };
2100
2101 static struct unit_test_suite bbdev_validation_testsuite = {
2102         .suite_name = "BBdev Validation Tests",
2103         .setup = testsuite_setup,
2104         .teardown = testsuite_teardown,
2105         .unit_test_cases = {
2106                 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc),
2107                 TEST_CASES_END() /**< NULL terminate unit test array */
2108         }
2109 };
2110
2111 static struct unit_test_suite bbdev_latency_testsuite = {
2112         .suite_name = "BBdev Latency Tests",
2113         .setup = testsuite_setup,
2114         .teardown = testsuite_teardown,
2115         .unit_test_cases = {
2116                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_tc),
2117                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
2118                 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc),
2119                 TEST_CASES_END() /**< NULL terminate unit test array */
2120         }
2121 };
2122
2123 static struct unit_test_suite bbdev_interrupt_testsuite = {
2124         .suite_name = "BBdev Interrupt Tests",
2125         .setup = interrupt_testsuite_setup,
2126         .teardown = testsuite_teardown,
2127         .unit_test_cases = {
2128                 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
2129                 TEST_CASES_END() /**< NULL terminate unit test array */
2130         }
2131 };
2132
2133 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
2134 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
2135 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
2136 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);