app/test-bbdev/test_bbdev_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7 #include <math.h>
   8
   9 #include <rte_eal.h>
  10 #include <rte_common.h>
  11 #include <rte_dev.h>
  12 #include <rte_launch.h>
  13 #include <rte_bbdev.h>
  14 #include <rte_cycles.h>
  15 #include <rte_lcore.h>
  16 #include <rte_malloc.h>
  17 #include <rte_random.h>
  18 #include <rte_hexdump.h>
  19
  20 #include "main.h"
  21 #include "test_bbdev_vector.h"
  22
  23 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
  24
  25 #define MAX_QUEUES RTE_MAX_LCORE
  26
  27 #define OPS_CACHE_SIZE 256U
  28 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
  29
  30 #define SYNC_WAIT 0
  31 #define SYNC_START 1
  32
  33 #define INVALID_QUEUE_ID -1
  34
  35 static struct test_bbdev_vector test_vector;
  36
  37 /* Switch between PMD and Interrupt for throughput TC */
  38 static bool intr_enabled;
  39
  40 /* Represents tested active devices */
  41 static struct active_device {
  42         const char *driver_name;
  43         uint8_t dev_id;
  44         uint16_t supported_ops;
  45         uint16_t queue_ids[MAX_QUEUES];
  46         uint16_t nb_queues;
  47         struct rte_mempool *ops_mempool;
  48         struct rte_mempool *in_mbuf_pool;
  49         struct rte_mempool *hard_out_mbuf_pool;
  50         struct rte_mempool *soft_out_mbuf_pool;
  51 } active_devs[RTE_BBDEV_MAX_DEVS];
  52
  53 static uint8_t nb_active_devs;
  54
  55 /* Data buffers used by BBDEV ops */
  56 struct test_buffers {
  57         struct rte_bbdev_op_data *inputs;
  58         struct rte_bbdev_op_data *hard_outputs;
  59         struct rte_bbdev_op_data *soft_outputs;
  60 };
  61
  62 /* Operation parameters specific for given test case */
  63 struct test_op_params {
  64         struct rte_mempool *mp;
  65         struct rte_bbdev_dec_op *ref_dec_op;
  66         struct rte_bbdev_enc_op *ref_enc_op;
  67         uint16_t burst_sz;
  68         uint16_t num_to_process;
  69         uint16_t num_lcores;
  70         int vector_mask;
  71         rte_atomic16_t sync;
  72         struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
  73 };
  74
  75 /* Contains per lcore params */
  76 struct thread_params {
  77         uint8_t dev_id;
  78         uint16_t queue_id;
  79         uint64_t start_time;
  80         double mops;
  81         double mbps;
  82         rte_atomic16_t nb_dequeued;
  83         rte_atomic16_t processing_status;
  84         struct test_op_params *op_params;
  85 };
  86
  87 typedef int (test_case_function)(struct active_device *ad,
  88                 struct test_op_params *op_params);
  89
  90 static inline void
  91 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
  92 {
  93         ad->supported_ops |= (1 << op_type);
  94 }
  95
  96 static inline bool
  97 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
  98 {
  99         return ad->supported_ops & (1 << op_type);
 100 }
 101
 102 static inline bool
 103 flags_match(uint32_t flags_req, uint32_t flags_present)
 104 {
 105         return (flags_req & flags_present) == flags_req;
 106 }
 107
 108 static void
 109 clear_soft_out_cap(uint32_t *op_flags)
 110 {
 111         *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
 112         *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
 113         *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
 114 }
 115
 116 static int
 117 check_dev_cap(const struct rte_bbdev_info *dev_info)
 118 {
 119         unsigned int i;
 120         unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs;
 121         const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
 122
 123         nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
 124         nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
 125         nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
 126
 127         for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
 128                 if (op_cap->type != test_vector.op_type)
 129                         continue;
 130
 131                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
 132                         const struct rte_bbdev_op_cap_turbo_dec *cap =
 133                                         &op_cap->cap.turbo_dec;
 134                         /* Ignore lack of soft output capability, just skip
 135                          * checking if soft output is valid.
 136                          */
 137                         if ((test_vector.turbo_dec.op_flags &
 138                                         RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
 139                                         !(cap->capability_flags &
 140                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 141                                 printf(
 142                                         "WARNING: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
 143                                         dev_info->dev_name);
 144                                 clear_soft_out_cap(
 145                                         &test_vector.turbo_dec.op_flags);
 146                         }
 147
 148                         if (!flags_match(test_vector.turbo_dec.op_flags,
 149                                         cap->capability_flags))
 150                                 return TEST_FAILED;
 151                         if (nb_inputs > cap->num_buffers_src) {
 152                                 printf("Too many inputs defined: %u, max: %u\n",
 153                                         nb_inputs, cap->num_buffers_src);
 154                                 return TEST_FAILED;
 155                         }
 156                         if (nb_soft_outputs > cap->num_buffers_soft_out &&
 157                                         (test_vector.turbo_dec.op_flags &
 158                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 159                                 printf(
 160                                         "Too many soft outputs defined: %u, max: %u\n",
 161                                                 nb_soft_outputs,
 162                                                 cap->num_buffers_soft_out);
 163                                 return TEST_FAILED;
 164                         }
 165                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 166                                 printf(
 167                                         "Too many hard outputs defined: %u, max: %u\n",
 168                                                 nb_hard_outputs,
 169                                                 cap->num_buffers_hard_out);
 170                                 return TEST_FAILED;
 171                         }
 172                         if (intr_enabled && !(cap->capability_flags &
 173                                         RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
 174                                 printf(
 175                                         "Dequeue interrupts are not supported!\n");
 176                                 return TEST_FAILED;
 177                         }
 178
 179                         return TEST_SUCCESS;
 180                 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
 181                         const struct rte_bbdev_op_cap_turbo_enc *cap =
 182                                         &op_cap->cap.turbo_enc;
 183
 184                         if (!flags_match(test_vector.turbo_enc.op_flags,
 185                                         cap->capability_flags))
 186                                 return TEST_FAILED;
 187                         if (nb_inputs > cap->num_buffers_src) {
 188                                 printf("Too many inputs defined: %u, max: %u\n",
 189                                         nb_inputs, cap->num_buffers_src);
 190                                 return TEST_FAILED;
 191                         }
 192                         if (nb_hard_outputs > cap->num_buffers_dst) {
 193                                 printf(
 194                                         "Too many hard outputs defined: %u, max: %u\n",
 195                                         nb_hard_outputs, cap->num_buffers_src);
 196                                 return TEST_FAILED;
 197                         }
 198                         if (intr_enabled && !(cap->capability_flags &
 199                                         RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
 200                                 printf(
 201                                         "Dequeue interrupts are not supported!\n");
 202                                 return TEST_FAILED;
 203                         }
 204
 205                         return TEST_SUCCESS;
 206                 }
 207         }
 208
 209         if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
 210                 return TEST_SUCCESS; /* Special case for NULL device */
 211
 212         return TEST_FAILED;
 213 }
 214
 215 /* calculates optimal mempool size not smaller than the val */
 216 static unsigned int
 217 optimal_mempool_size(unsigned int val)
 218 {
 219         return rte_align32pow2(val + 1) - 1;
 220 }
 221
 222 /* allocates mbuf mempool for inputs and outputs */
 223 static struct rte_mempool *
 224 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
 225                 int socket_id, unsigned int mbuf_pool_size,
 226                 const char *op_type_str)
 227 {
 228         unsigned int i;
 229         uint32_t max_seg_sz = 0;
 230         char pool_name[RTE_MEMPOOL_NAMESIZE];
 231
 232         /* find max input segment size */
 233         for (i = 0; i < entries->nb_segments; ++i)
 234                 if (entries->segments[i].length > max_seg_sz)
 235                         max_seg_sz = entries->segments[i].length;
 236
 237         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 238                         dev_id);
 239         return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
 240                         RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM,
 241                         (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
 242 }
 243
 244 static int
 245 create_mempools(struct active_device *ad, int socket_id,
 246                 enum rte_bbdev_op_type op_type, uint16_t num_ops)
 247 {
 248         struct rte_mempool *mp;
 249         unsigned int ops_pool_size, mbuf_pool_size = 0;
 250         char pool_name[RTE_MEMPOOL_NAMESIZE];
 251         const char *op_type_str;
 252
 253         struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
 254         struct op_data_entries *hard_out =
 255                         &test_vector.entries[DATA_HARD_OUTPUT];
 256         struct op_data_entries *soft_out =
 257                         &test_vector.entries[DATA_SOFT_OUTPUT];
 258
 259         /* allocate ops mempool */
 260         ops_pool_size = optimal_mempool_size(RTE_MAX(
 261                         /* Ops used plus 1 reference op */
 262                         RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
 263                         /* Minimal cache size plus 1 reference op */
 264                         (unsigned int)(1.5 * rte_lcore_count() *
 265                                         OPS_CACHE_SIZE + 1)),
 266                         OPS_POOL_SIZE_MIN));
 267
 268         op_type_str = rte_bbdev_op_type_str(op_type);
 269         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
 270
 271         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 272                         ad->dev_id);
 273         mp = rte_bbdev_op_pool_create(pool_name, op_type,
 274                         ops_pool_size, OPS_CACHE_SIZE, socket_id);
 275         TEST_ASSERT_NOT_NULL(mp,
 276                         "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
 277                         ops_pool_size,
 278                         ad->dev_id,
 279                         socket_id);
 280         ad->ops_mempool = mp;
 281
 282         /* Inputs */
 283         mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments);
 284         mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in");
 285         TEST_ASSERT_NOT_NULL(mp,
 286                         "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
 287                         mbuf_pool_size,
 288                         ad->dev_id,
 289                         socket_id);
 290         ad->in_mbuf_pool = mp;
 291
 292         /* Hard outputs */
 293         mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 294                         hard_out->nb_segments);
 295         mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size,
 296                         "hard_out");
 297         TEST_ASSERT_NOT_NULL(mp,
 298                         "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
 299                         mbuf_pool_size,
 300                         ad->dev_id,
 301                         socket_id);
 302         ad->hard_out_mbuf_pool = mp;
 303
 304         if (soft_out->nb_segments == 0)
 305                 return TEST_SUCCESS;
 306
 307         /* Soft outputs */
 308         mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 309                         soft_out->nb_segments);
 310         mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, mbuf_pool_size,
 311                         "soft_out");
 312         TEST_ASSERT_NOT_NULL(mp,
 313                         "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
 314                         mbuf_pool_size,
 315                         ad->dev_id,
 316                         socket_id);
 317         ad->soft_out_mbuf_pool = mp;
 318
 319         return 0;
 320 }
 321
 322 static int
 323 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
 324                 struct test_bbdev_vector *vector)
 325 {
 326         int ret;
 327         unsigned int queue_id;
 328         struct rte_bbdev_queue_conf qconf;
 329         struct active_device *ad = &active_devs[nb_active_devs];
 330         unsigned int nb_queues;
 331         enum rte_bbdev_op_type op_type = vector->op_type;
 332
 333         nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
 334         /* setup device */
 335         ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
 336         if (ret < 0) {
 337                 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
 338                                 dev_id, nb_queues, info->socket_id, ret);
 339                 return TEST_FAILED;
 340         }
 341
 342         /* configure interrupts if needed */
 343         if (intr_enabled) {
 344                 ret = rte_bbdev_intr_enable(dev_id);
 345                 if (ret < 0) {
 346                         printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
 347                                         ret);
 348                         return TEST_FAILED;
 349                 }
 350         }
 351
 352         /* setup device queues */
 353         qconf.socket = info->socket_id;
 354         qconf.queue_size = info->drv.default_queue_conf.queue_size;
 355         qconf.priority = 0;
 356         qconf.deferred_start = 0;
 357         qconf.op_type = op_type;
 358
 359         for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
 360                 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
 361                 if (ret != 0) {
 362                         printf(
 363                                         "Allocated all queues (id=%u) at prio%u on dev%u\n",
 364                                         queue_id, qconf.priority, dev_id);
 365                         qconf.priority++;
 366                         ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
 367                                         &qconf);
 368                 }
 369                 if (ret != 0) {
 370                         printf("All queues on dev %u allocated: %u\n",
 371                                         dev_id, queue_id);
 372                         break;
 373                 }
 374                 ad->queue_ids[queue_id] = queue_id;
 375         }
 376         TEST_ASSERT(queue_id != 0,
 377                         "ERROR Failed to configure any queues on dev %u",
 378                         dev_id);
 379         ad->nb_queues = queue_id;
 380
 381         set_avail_op(ad, op_type);
 382
 383         return TEST_SUCCESS;
 384 }
 385
 386 static int
 387 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
 388                 struct test_bbdev_vector *vector)
 389 {
 390         int ret;
 391
 392         active_devs[nb_active_devs].driver_name = info->drv.driver_name;
 393         active_devs[nb_active_devs].dev_id = dev_id;
 394
 395         ret = add_bbdev_dev(dev_id, info, vector);
 396         if (ret == TEST_SUCCESS)
 397                 ++nb_active_devs;
 398         return ret;
 399 }
 400
 401 static uint8_t
 402 populate_active_devices(void)
 403 {
 404         int ret;
 405         uint8_t dev_id;
 406         uint8_t nb_devs_added = 0;
 407         struct rte_bbdev_info info;
 408
 409         RTE_BBDEV_FOREACH(dev_id) {
 410                 rte_bbdev_info_get(dev_id, &info);
 411
 412                 if (check_dev_cap(&info)) {
 413                         printf(
 414                                 "Device %d (%s) does not support specified capabilities\n",
 415                                         dev_id, info.dev_name);
 416                         continue;
 417                 }
 418
 419                 ret = add_active_device(dev_id, &info, &test_vector);
 420                 if (ret != 0) {
 421                         printf("Adding active bbdev %s skipped\n",
 422                                         info.dev_name);
 423                         continue;
 424                 }
 425                 nb_devs_added++;
 426         }
 427
 428         return nb_devs_added;
 429 }
 430
 431 static int
 432 read_test_vector(void)
 433 {
 434         int ret;
 435
 436         memset(&test_vector, 0, sizeof(test_vector));
 437         printf("Test vector file = %s\n", get_vector_filename());
 438         ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
 439         TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
 440                         get_vector_filename());
 441
 442         return TEST_SUCCESS;
 443 }
 444
 445 static int
 446 testsuite_setup(void)
 447 {
 448         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 449
 450         if (populate_active_devices() == 0) {
 451                 printf("No suitable devices found!\n");
 452                 return TEST_SKIPPED;
 453         }
 454
 455         return TEST_SUCCESS;
 456 }
 457
 458 static int
 459 interrupt_testsuite_setup(void)
 460 {
 461         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 462
 463         /* Enable interrupts */
 464         intr_enabled = true;
 465
 466         /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
 467         if (populate_active_devices() == 0 ||
 468                         test_vector.op_type == RTE_BBDEV_OP_NONE) {
 469                 intr_enabled = false;
 470                 printf("No suitable devices found!\n");
 471                 return TEST_SKIPPED;
 472         }
 473
 474         return TEST_SUCCESS;
 475 }
 476
 477 static void
 478 testsuite_teardown(void)
 479 {
 480         uint8_t dev_id;
 481
 482         /* Unconfigure devices */
 483         RTE_BBDEV_FOREACH(dev_id)
 484                 rte_bbdev_close(dev_id);
 485
 486         /* Clear active devices structs. */
 487         memset(active_devs, 0, sizeof(active_devs));
 488         nb_active_devs = 0;
 489 }
 490
 491 static int
 492 ut_setup(void)
 493 {
 494         uint8_t i, dev_id;
 495
 496         for (i = 0; i < nb_active_devs; i++) {
 497                 dev_id = active_devs[i].dev_id;
 498                 /* reset bbdev stats */
 499                 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
 500                                 "Failed to reset stats of bbdev %u", dev_id);
 501                 /* start the device */
 502                 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
 503                                 "Failed to start bbdev %u", dev_id);
 504         }
 505
 506         return TEST_SUCCESS;
 507 }
 508
 509 static void
 510 ut_teardown(void)
 511 {
 512         uint8_t i, dev_id;
 513         struct rte_bbdev_stats stats;
 514
 515         for (i = 0; i < nb_active_devs; i++) {
 516                 dev_id = active_devs[i].dev_id;
 517                 /* read stats and print */
 518                 rte_bbdev_stats_get(dev_id, &stats);
 519                 /* Stop the device */
 520                 rte_bbdev_stop(dev_id);
 521         }
 522 }
 523
 524 static int
 525 init_op_data_objs(struct rte_bbdev_op_data *bufs,
 526                 struct op_data_entries *ref_entries,
 527                 struct rte_mempool *mbuf_pool, const uint16_t n,
 528                 enum op_data_type op_type, uint16_t min_alignment)
 529 {
 530         int ret;
 531         unsigned int i, j;
 532
 533         for (i = 0; i < n; ++i) {
 534                 char *data;
 535                 struct op_data_buf *seg = &ref_entries->segments[0];
 536                 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
 537                 TEST_ASSERT_NOT_NULL(m_head,
 538                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 539                                 op_type, n * ref_entries->nb_segments,
 540                                 mbuf_pool->size);
 541
 542                 bufs[i].data = m_head;
 543                 bufs[i].offset = 0;
 544                 bufs[i].length = 0;
 545
 546                 if (op_type == DATA_INPUT) {
 547                         data = rte_pktmbuf_append(m_head, seg->length);
 548                         TEST_ASSERT_NOT_NULL(data,
 549                                         "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 550                                         seg->length, op_type);
 551
 552                         TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment),
 553                                         "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 554                                         data, min_alignment);
 555                         rte_memcpy(data, seg->addr, seg->length);
 556                         bufs[i].length += seg->length;
 557
 558
 559                         for (j = 1; j < ref_entries->nb_segments; ++j) {
 560                                 struct rte_mbuf *m_tail =
 561                                                 rte_pktmbuf_alloc(mbuf_pool);
 562                                 TEST_ASSERT_NOT_NULL(m_tail,
 563                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 564                                                 op_type,
 565                                                 n * ref_entries->nb_segments,
 566                                                 mbuf_pool->size);
 567                                 seg += 1;
 568
 569                                 data = rte_pktmbuf_append(m_tail, seg->length);
 570                                 TEST_ASSERT_NOT_NULL(data,
 571                                                 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 572                                                 seg->length, op_type);
 573
 574                                 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
 575                                                 min_alignment),
 576                                                 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 577                                                 data, min_alignment);
 578                                 rte_memcpy(data, seg->addr, seg->length);
 579                                 bufs[i].length += seg->length;
 580
 581                                 ret = rte_pktmbuf_chain(m_head, m_tail);
 582                                 TEST_ASSERT_SUCCESS(ret,
 583                                                 "Couldn't chain mbufs from %d data type mbuf pool",
 584                                                 op_type);
 585                         }
 586                 }
 587         }
 588
 589         return 0;
 590 }
 591
 592 static int
 593 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
 594                 const int socket)
 595 {
 596         int i;
 597
 598         *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
 599         if (*buffers == NULL) {
 600                 printf("WARNING: Failed to allocate op_data on socket %d\n",
 601                                 socket);
 602                 /* try to allocate memory on other detected sockets */
 603                 for (i = 0; i < socket; i++) {
 604                         *buffers = rte_zmalloc_socket(NULL, len, 0, i);
 605                         if (*buffers != NULL)
 606                                 break;
 607                 }
 608         }
 609
 610         return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
 611 }
 612
 613 static void
 614 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
 615                 uint16_t n, int8_t max_llr_modulus)
 616 {
 617         uint16_t i, byte_idx;
 618
 619         for (i = 0; i < n; ++i) {
 620                 struct rte_mbuf *m = input_ops[i].data;
 621                 while (m != NULL) {
 622                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
 623                                         input_ops[i].offset);
 624                         for (byte_idx = 0; byte_idx < input_ops[i].length;
 625                                         ++byte_idx)
 626                                 llr[byte_idx] = round((double)max_llr_modulus *
 627                                                 llr[byte_idx] / INT8_MAX);
 628
 629                         m = m->next;
 630                 }
 631         }
 632 }
 633
 634 static int
 635 fill_queue_buffers(struct test_op_params *op_params,
 636                 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
 637                 struct rte_mempool *soft_out_mp, uint16_t queue_id,
 638                 const struct rte_bbdev_op_cap *capabilities,
 639                 uint16_t min_alignment, const int socket_id)
 640 {
 641         int ret;
 642         enum op_data_type type;
 643         const uint16_t n = op_params->num_to_process;
 644
 645         struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
 646                 in_mp,
 647                 soft_out_mp,
 648                 hard_out_mp,
 649         };
 650
 651         struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
 652                 &op_params->q_bufs[socket_id][queue_id].inputs,
 653                 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
 654                 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
 655         };
 656
 657         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
 658                 struct op_data_entries *ref_entries =
 659                                 &test_vector.entries[type];
 660                 if (ref_entries->nb_segments == 0)
 661                         continue;
 662
 663                 ret = allocate_buffers_on_socket(queue_ops[type],
 664                                 n * sizeof(struct rte_bbdev_op_data),
 665                                 socket_id);
 666                 TEST_ASSERT_SUCCESS(ret,
 667                                 "Couldn't allocate memory for rte_bbdev_op_data structs");
 668
 669                 ret = init_op_data_objs(*queue_ops[type], ref_entries,
 670                                 mbuf_pools[type], n, type, min_alignment);
 671                 TEST_ASSERT_SUCCESS(ret,
 672                                 "Couldn't init rte_bbdev_op_data structs");
 673         }
 674
 675         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
 676                 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
 677                         capabilities->cap.turbo_dec.max_llr_modulus);
 678
 679         return 0;
 680 }
 681
 682 static void
 683 free_buffers(struct active_device *ad, struct test_op_params *op_params)
 684 {
 685         unsigned int i, j;
 686
 687         rte_mempool_free(ad->ops_mempool);
 688         rte_mempool_free(ad->in_mbuf_pool);
 689         rte_mempool_free(ad->hard_out_mbuf_pool);
 690         rte_mempool_free(ad->soft_out_mbuf_pool);
 691
 692         for (i = 0; i < rte_lcore_count(); ++i) {
 693                 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
 694                         rte_free(op_params->q_bufs[j][i].inputs);
 695                         rte_free(op_params->q_bufs[j][i].hard_outputs);
 696                         rte_free(op_params->q_bufs[j][i].soft_outputs);
 697                 }
 698         }
 699 }
 700
 701 static void
 702 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
 703                 unsigned int start_idx,
 704                 struct rte_bbdev_op_data *inputs,
 705                 struct rte_bbdev_op_data *hard_outputs,
 706                 struct rte_bbdev_op_data *soft_outputs,
 707                 struct rte_bbdev_dec_op *ref_op)
 708 {
 709         unsigned int i;
 710         struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
 711
 712         for (i = 0; i < n; ++i) {
 713                 if (turbo_dec->code_block_mode == 0) {
 714                         ops[i]->turbo_dec.tb_params.ea =
 715                                         turbo_dec->tb_params.ea;
 716                         ops[i]->turbo_dec.tb_params.eb =
 717                                         turbo_dec->tb_params.eb;
 718                         ops[i]->turbo_dec.tb_params.k_pos =
 719                                         turbo_dec->tb_params.k_pos;
 720                         ops[i]->turbo_dec.tb_params.k_neg =
 721                                         turbo_dec->tb_params.k_neg;
 722                         ops[i]->turbo_dec.tb_params.c =
 723                                         turbo_dec->tb_params.c;
 724                         ops[i]->turbo_dec.tb_params.c_neg =
 725                                         turbo_dec->tb_params.c_neg;
 726                         ops[i]->turbo_dec.tb_params.cab =
 727                                         turbo_dec->tb_params.cab;
 728                 } else {
 729                         ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
 730                         ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
 731                 }
 732
 733                 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
 734                 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
 735                 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
 736                 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
 737                 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
 738                 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
 739                 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
 740
 741                 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
 742                 ops[i]->turbo_dec.input = inputs[start_idx + i];
 743                 if (soft_outputs != NULL)
 744                         ops[i]->turbo_dec.soft_output =
 745                                 soft_outputs[start_idx + i];
 746         }
 747 }
 748
 749 static void
 750 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
 751                 unsigned int start_idx,
 752                 struct rte_bbdev_op_data *inputs,
 753                 struct rte_bbdev_op_data *outputs,
 754                 struct rte_bbdev_enc_op *ref_op)
 755 {
 756         unsigned int i;
 757         struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
 758         for (i = 0; i < n; ++i) {
 759                 if (turbo_enc->code_block_mode == 0) {
 760                         ops[i]->turbo_enc.tb_params.ea =
 761                                         turbo_enc->tb_params.ea;
 762                         ops[i]->turbo_enc.tb_params.eb =
 763                                         turbo_enc->tb_params.eb;
 764                         ops[i]->turbo_enc.tb_params.k_pos =
 765                                         turbo_enc->tb_params.k_pos;
 766                         ops[i]->turbo_enc.tb_params.k_neg =
 767                                         turbo_enc->tb_params.k_neg;
 768                         ops[i]->turbo_enc.tb_params.c =
 769                                         turbo_enc->tb_params.c;
 770                         ops[i]->turbo_enc.tb_params.c_neg =
 771                                         turbo_enc->tb_params.c_neg;
 772                         ops[i]->turbo_enc.tb_params.cab =
 773                                         turbo_enc->tb_params.cab;
 774                         ops[i]->turbo_enc.tb_params.ncb_pos =
 775                                         turbo_enc->tb_params.ncb_pos;
 776                         ops[i]->turbo_enc.tb_params.ncb_neg =
 777                                         turbo_enc->tb_params.ncb_neg;
 778                         ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
 779                 } else {
 780                         ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
 781                         ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
 782                         ops[i]->turbo_enc.cb_params.ncb =
 783                                         turbo_enc->cb_params.ncb;
 784                 }
 785                 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
 786                 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
 787                 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
 788
 789                 ops[i]->turbo_enc.output = outputs[start_idx + i];
 790                 ops[i]->turbo_enc.input = inputs[start_idx + i];
 791         }
 792 }
 793
 794 static int
 795 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
 796                 unsigned int order_idx, const int expected_status)
 797 {
 798         TEST_ASSERT(op->status == expected_status,
 799                         "op_status (%d) != expected_status (%d)",
 800                         op->status, expected_status);
 801
 802         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
 803                         "Ordering error, expected %p, got %p",
 804                         (void *)(uintptr_t)order_idx, op->opaque_data);
 805
 806         return TEST_SUCCESS;
 807 }
 808
 809 static int
 810 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
 811                 unsigned int order_idx, const int expected_status)
 812 {
 813         TEST_ASSERT(op->status == expected_status,
 814                         "op_status (%d) != expected_status (%d)",
 815                         op->status, expected_status);
 816
 817         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
 818                         "Ordering error, expected %p, got %p",
 819                         (void *)(uintptr_t)order_idx, op->opaque_data);
 820
 821         return TEST_SUCCESS;
 822 }
 823
 824 static inline int
 825 validate_op_chain(struct rte_bbdev_op_data *op,
 826                 struct op_data_entries *orig_op)
 827 {
 828         uint8_t i;
 829         struct rte_mbuf *m = op->data;
 830         uint8_t nb_dst_segments = orig_op->nb_segments;
 831
 832         TEST_ASSERT(nb_dst_segments == m->nb_segs,
 833                         "Number of segments differ in original (%u) and filled (%u) op",
 834                         nb_dst_segments, m->nb_segs);
 835
 836         for (i = 0; i < nb_dst_segments; ++i) {
 837                 /* Apply offset to the first mbuf segment */
 838                 uint16_t offset = (i == 0) ? op->offset : 0;
 839                 uint16_t data_len = m->data_len - offset;
 840
 841                 TEST_ASSERT(orig_op->segments[i].length == data_len,
 842                                 "Length of segment differ in original (%u) and filled (%u) op",
 843                                 orig_op->segments[i].length, data_len);
 844                 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
 845                                 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
 846                                 data_len,
 847                                 "Output buffers (CB=%u) are not equal", i);
 848                 m = m->next;
 849         }
 850
 851         return TEST_SUCCESS;
 852 }
 853
 854 static int
 855 validate_dec_buffers(struct rte_bbdev_dec_op *ref_op, struct test_buffers *bufs,
 856                 const uint16_t num_to_process)
 857 {
 858         int i;
 859
 860         struct op_data_entries *hard_data_orig =
 861                         &test_vector.entries[DATA_HARD_OUTPUT];
 862         struct op_data_entries *soft_data_orig =
 863                         &test_vector.entries[DATA_SOFT_OUTPUT];
 864
 865         for (i = 0; i < num_to_process; i++) {
 866                 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i],
 867                                 hard_data_orig),
 868                                 "Hard output buffers are not equal");
 869                 if (ref_op->turbo_dec.op_flags &
 870                                 RTE_BBDEV_TURBO_SOFT_OUTPUT)
 871                         TEST_ASSERT_SUCCESS(validate_op_chain(
 872                                         &bufs->soft_outputs[i],
 873                                         soft_data_orig),
 874                                         "Soft output buffers are not equal");
 875         }
 876
 877         return TEST_SUCCESS;
 878 }
 879
 880 static int
 881 validate_enc_buffers(struct test_buffers *bufs, const uint16_t num_to_process)
 882 {
 883         int i;
 884
 885         struct op_data_entries *hard_data_orig =
 886                         &test_vector.entries[DATA_HARD_OUTPUT];
 887
 888         for (i = 0; i < num_to_process; i++)
 889                 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i],
 890                                 hard_data_orig), "");
 891
 892         return TEST_SUCCESS;
 893 }
 894
 895 static int
 896 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
 897                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
 898 {
 899         unsigned int i;
 900         int ret;
 901         struct op_data_entries *hard_data_orig =
 902                         &test_vector.entries[DATA_HARD_OUTPUT];
 903         struct op_data_entries *soft_data_orig =
 904                         &test_vector.entries[DATA_SOFT_OUTPUT];
 905         struct rte_bbdev_op_turbo_dec *ops_td;
 906         struct rte_bbdev_op_data *hard_output;
 907         struct rte_bbdev_op_data *soft_output;
 908         struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
 909
 910         for (i = 0; i < n; ++i) {
 911                 ops_td = &ops[i]->turbo_dec;
 912                 hard_output = &ops_td->hard_output;
 913                 soft_output = &ops_td->soft_output;
 914
 915                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
 916                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
 917                                         "Returned iter_count (%d) > expected iter_count (%d)",
 918                                         ops_td->iter_count, ref_td->iter_count);
 919                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
 920                 TEST_ASSERT_SUCCESS(ret,
 921                                 "Checking status and ordering for decoder failed");
 922
 923                 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
 924                                 hard_data_orig),
 925                                 "Hard output buffers (CB=%u) are not equal",
 926                                 i);
 927
 928                 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
 929                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
 930                                         soft_data_orig),
 931                                         "Soft output buffers (CB=%u) are not equal",
 932                                         i);
 933         }
 934
 935         return TEST_SUCCESS;
 936 }
 937
 938 static int
 939 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
 940                 struct rte_bbdev_enc_op *ref_op)
 941 {
 942         unsigned int i;
 943         int ret;
 944         struct op_data_entries *hard_data_orig =
 945                         &test_vector.entries[DATA_HARD_OUTPUT];
 946
 947         for (i = 0; i < n; ++i) {
 948                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
 949                 TEST_ASSERT_SUCCESS(ret,
 950                                 "Checking status and ordering for encoder failed");
 951                 TEST_ASSERT_SUCCESS(validate_op_chain(
 952                                 &ops[i]->turbo_enc.output,
 953                                 hard_data_orig),
 954                                 "Output buffers (CB=%u) are not equal",
 955                                 i);
 956         }
 957
 958         return TEST_SUCCESS;
 959 }
 960
 961 static void
 962 create_reference_dec_op(struct rte_bbdev_dec_op *op)
 963 {
 964         unsigned int i;
 965         struct op_data_entries *entry;
 966
 967         op->turbo_dec = test_vector.turbo_dec;
 968         entry = &test_vector.entries[DATA_INPUT];
 969         for (i = 0; i < entry->nb_segments; ++i)
 970                 op->turbo_dec.input.length +=
 971                                 entry->segments[i].length;
 972 }
 973
 974 static void
 975 create_reference_enc_op(struct rte_bbdev_enc_op *op)
 976 {
 977         unsigned int i;
 978         struct op_data_entries *entry;
 979
 980         op->turbo_enc = test_vector.turbo_enc;
 981         entry = &test_vector.entries[DATA_INPUT];
 982         for (i = 0; i < entry->nb_segments; ++i)
 983                 op->turbo_enc.input.length +=
 984                                 entry->segments[i].length;
 985 }
 986
 987 static int
 988 init_test_op_params(struct test_op_params *op_params,
 989                 enum rte_bbdev_op_type op_type, const int expected_status,
 990                 const int vector_mask, struct rte_mempool *ops_mp,
 991                 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
 992 {
 993         int ret = 0;
 994         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
 995                 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
 996                                 &op_params->ref_dec_op, 1);
 997         else
 998                 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
 999                                 &op_params->ref_enc_op, 1);
1000
1001         TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
1002
1003         op_params->mp = ops_mp;
1004         op_params->burst_sz = burst_sz;
1005         op_params->num_to_process = num_to_process;
1006         op_params->num_lcores = num_lcores;
1007         op_params->vector_mask = vector_mask;
1008         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1009                 op_params->ref_dec_op->status = expected_status;
1010         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
1011                 op_params->ref_enc_op->status = expected_status;
1012
1013         return 0;
1014 }
1015
1016 static int
1017 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
1018                 struct test_op_params *op_params)
1019 {
1020         int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
1021         unsigned int i;
1022         struct active_device *ad;
1023         unsigned int burst_sz = get_burst_sz();
1024         enum rte_bbdev_op_type op_type = test_vector.op_type;
1025         const struct rte_bbdev_op_cap *capabilities = NULL;
1026
1027         ad = &active_devs[dev_id];
1028
1029         /* Check if device supports op_type */
1030         if (!is_avail_op(ad, test_vector.op_type))
1031                 return TEST_SUCCESS;
1032
1033         struct rte_bbdev_info info;
1034         rte_bbdev_info_get(ad->dev_id, &info);
1035         socket_id = GET_SOCKET(info.socket_id);
1036
1037         if (op_type == RTE_BBDEV_OP_NONE)
1038                 op_type = RTE_BBDEV_OP_TURBO_ENC;
1039         f_ret = create_mempools(ad, socket_id, op_type,
1040                         get_num_ops());
1041         if (f_ret != TEST_SUCCESS) {
1042                 printf("Couldn't create mempools");
1043                 goto fail;
1044         }
1045
1046         f_ret = init_test_op_params(op_params, test_vector.op_type,
1047                         test_vector.expected_status,
1048                         test_vector.mask,
1049                         ad->ops_mempool,
1050                         burst_sz,
1051                         get_num_ops(),
1052                         get_num_lcores());
1053         if (f_ret != TEST_SUCCESS) {
1054                 printf("Couldn't init test op params");
1055                 goto fail;
1056         }
1057
1058         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
1059                 /* Find Decoder capabilities */
1060                 const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
1061                 while (cap->type != RTE_BBDEV_OP_NONE) {
1062                         if (cap->type == RTE_BBDEV_OP_TURBO_DEC) {
1063                                 capabilities = cap;
1064                                 break;
1065                         }
1066                 }
1067                 TEST_ASSERT_NOT_NULL(capabilities,
1068                                 "Couldn't find Decoder capabilities");
1069
1070                 create_reference_dec_op(op_params->ref_dec_op);
1071         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1072                 create_reference_enc_op(op_params->ref_enc_op);
1073
1074         for (i = 0; i < ad->nb_queues; ++i) {
1075                 f_ret = fill_queue_buffers(op_params,
1076                                 ad->in_mbuf_pool,
1077                                 ad->hard_out_mbuf_pool,
1078                                 ad->soft_out_mbuf_pool,
1079                                 ad->queue_ids[i],
1080                                 capabilities,
1081                                 info.drv.min_alignment,
1082                                 socket_id);
1083                 if (f_ret != TEST_SUCCESS) {
1084                         printf("Couldn't init queue buffers");
1085                         goto fail;
1086                 }
1087         }
1088
1089         /* Run test case function */
1090         t_ret = test_case_func(ad, op_params);
1091
1092         /* Free active device resources and return */
1093         free_buffers(ad, op_params);
1094         return t_ret;
1095
1096 fail:
1097         free_buffers(ad, op_params);
1098         return TEST_FAILED;
1099 }
1100
1101 /* Run given test function per active device per supported op type
1102  * per burst size.
1103  */
1104 static int
1105 run_test_case(test_case_function *test_case_func)
1106 {
1107         int ret = 0;
1108         uint8_t dev;
1109
1110         /* Alloc op_params */
1111         struct test_op_params *op_params = rte_zmalloc(NULL,
1112                         sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
1113         TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
1114                         RTE_ALIGN(sizeof(struct test_op_params),
1115                                 RTE_CACHE_LINE_SIZE));
1116
1117         /* For each device run test case function */
1118         for (dev = 0; dev < nb_active_devs; ++dev)
1119                 ret |= run_test_case_on_device(test_case_func, dev, op_params);
1120
1121         rte_free(op_params);
1122
1123         return ret;
1124 }
1125
1126 static void
1127 dequeue_event_callback(uint16_t dev_id,
1128                 enum rte_bbdev_event_type event, void *cb_arg,
1129                 void *ret_param)
1130 {
1131         int ret;
1132         uint16_t i;
1133         uint64_t total_time;
1134         uint16_t deq, burst_sz, num_to_process;
1135         uint16_t queue_id = INVALID_QUEUE_ID;
1136         struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
1137         struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
1138         struct test_buffers *bufs;
1139         struct rte_bbdev_info info;
1140
1141         /* Input length in bytes, million operations per second,
1142          * million bits per second.
1143          */
1144         double in_len;
1145
1146         struct thread_params *tp = cb_arg;
1147
1148         RTE_SET_USED(ret_param);
1149         queue_id = tp->queue_id;
1150
1151         /* Find matching thread params using queue_id */
1152         for (i = 0; i < MAX_QUEUES; ++i, ++tp)
1153                 if (tp->queue_id == queue_id)
1154                         break;
1155
1156         if (i == MAX_QUEUES) {
1157                 printf("%s: Queue_id from interrupt details was not found!\n",
1158                                 __func__);
1159                 return;
1160         }
1161
1162         if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
1163                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1164                 printf(
1165                         "Dequeue interrupt handler called for incorrect event!\n");
1166                 return;
1167         }
1168
1169         burst_sz = tp->op_params->burst_sz;
1170         num_to_process = tp->op_params->num_to_process;
1171
1172         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1173                 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, dec_ops,
1174                                 burst_sz);
1175         else
1176                 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, enc_ops,
1177                                 burst_sz);
1178
1179         if (deq < burst_sz) {
1180                 printf(
1181                         "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
1182                         burst_sz, deq);
1183                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1184                 return;
1185         }
1186
1187         if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_to_process) {
1188                 rte_atomic16_add(&tp->nb_dequeued, deq);
1189                 return;
1190         }
1191
1192         total_time = rte_rdtsc_precise() - tp->start_time;
1193
1194         rte_bbdev_info_get(dev_id, &info);
1195
1196         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1197
1198         ret = TEST_SUCCESS;
1199         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1200                 ret = validate_dec_buffers(tp->op_params->ref_dec_op, bufs,
1201                                 num_to_process);
1202         else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1203                 ret = validate_enc_buffers(bufs, num_to_process);
1204
1205         if (ret) {
1206                 printf("Buffers validation failed\n");
1207                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1208         }
1209
1210         switch (test_vector.op_type) {
1211         case RTE_BBDEV_OP_TURBO_DEC:
1212                 in_len = tp->op_params->ref_dec_op->turbo_dec.input.length;
1213                 break;
1214         case RTE_BBDEV_OP_TURBO_ENC:
1215                 in_len = tp->op_params->ref_enc_op->turbo_enc.input.length;
1216                 break;
1217         case RTE_BBDEV_OP_NONE:
1218                 in_len = 0.0;
1219                 break;
1220         default:
1221                 printf("Unknown op type: %d\n", test_vector.op_type);
1222                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1223                 return;
1224         }
1225
1226         tp->mops = ((double)num_to_process / 1000000.0) /
1227                         ((double)total_time / (double)rte_get_tsc_hz());
1228         tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1229                         ((double)total_time / (double)rte_get_tsc_hz());
1230
1231         rte_atomic16_add(&tp->nb_dequeued, deq);
1232 }
1233
1234 static int
1235 throughput_intr_lcore_dec(void *arg)
1236 {
1237         struct thread_params *tp = arg;
1238         unsigned int enqueued;
1239         struct rte_bbdev_dec_op *ops[MAX_BURST];
1240         const uint16_t queue_id = tp->queue_id;
1241         const uint16_t burst_sz = tp->op_params->burst_sz;
1242         const uint16_t num_to_process = tp->op_params->num_to_process;
1243         struct test_buffers *bufs = NULL;
1244         unsigned int allocs_failed = 0;
1245         struct rte_bbdev_info info;
1246         int ret;
1247
1248         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1249                         "BURST_SIZE should be <= %u", MAX_BURST);
1250
1251         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1252                         "Failed to enable interrupts for dev: %u, queue_id: %u",
1253                         tp->dev_id, queue_id);
1254
1255         rte_bbdev_info_get(tp->dev_id, &info);
1256         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1257
1258         rte_atomic16_clear(&tp->processing_status);
1259         rte_atomic16_clear(&tp->nb_dequeued);
1260
1261         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1262                 rte_pause();
1263
1264         tp->start_time = rte_rdtsc_precise();
1265         for (enqueued = 0; enqueued < num_to_process;) {
1266
1267                 uint16_t num_to_enq = burst_sz;
1268
1269                 if (unlikely(num_to_process - enqueued < num_to_enq))
1270                         num_to_enq = num_to_process - enqueued;
1271
1272                 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
1273                                 num_to_enq);
1274                 if (ret != 0) {
1275                         allocs_failed++;
1276                         continue;
1277                 }
1278
1279                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1280                         copy_reference_dec_op(ops, num_to_enq, enqueued,
1281                                         bufs->inputs,
1282                                         bufs->hard_outputs,
1283                                         bufs->soft_outputs,
1284                                         tp->op_params->ref_dec_op);
1285
1286                 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, queue_id, ops,
1287                                 num_to_enq);
1288
1289                 rte_bbdev_dec_op_free_bulk(ops, num_to_enq);
1290         }
1291
1292         if (allocs_failed > 0)
1293                 printf("WARNING: op allocations failed: %u times\n",
1294                                 allocs_failed);
1295
1296         return TEST_SUCCESS;
1297 }
1298
1299 static int
1300 throughput_intr_lcore_enc(void *arg)
1301 {
1302         struct thread_params *tp = arg;
1303         unsigned int enqueued;
1304         struct rte_bbdev_enc_op *ops[MAX_BURST];
1305         const uint16_t queue_id = tp->queue_id;
1306         const uint16_t burst_sz = tp->op_params->burst_sz;
1307         const uint16_t num_to_process = tp->op_params->num_to_process;
1308         struct test_buffers *bufs = NULL;
1309         unsigned int allocs_failed = 0;
1310         struct rte_bbdev_info info;
1311         int ret;
1312
1313         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1314                         "BURST_SIZE should be <= %u", MAX_BURST);
1315
1316         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1317                         "Failed to enable interrupts for dev: %u, queue_id: %u",
1318                         tp->dev_id, queue_id);
1319
1320         rte_bbdev_info_get(tp->dev_id, &info);
1321         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1322
1323         rte_atomic16_clear(&tp->processing_status);
1324         rte_atomic16_clear(&tp->nb_dequeued);
1325
1326         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1327                 rte_pause();
1328
1329         tp->start_time = rte_rdtsc_precise();
1330         for (enqueued = 0; enqueued < num_to_process;) {
1331
1332                 uint16_t num_to_enq = burst_sz;
1333
1334                 if (unlikely(num_to_process - enqueued < num_to_enq))
1335                         num_to_enq = num_to_process - enqueued;
1336
1337                 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
1338                                 num_to_enq);
1339                 if (ret != 0) {
1340                         allocs_failed++;
1341                         continue;
1342                 }
1343
1344                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1345                         copy_reference_enc_op(ops, num_to_enq, enqueued,
1346                                         bufs->inputs,
1347                                         bufs->hard_outputs,
1348                                         tp->op_params->ref_enc_op);
1349
1350                 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, queue_id, ops,
1351                                 num_to_enq);
1352
1353                 rte_bbdev_enc_op_free_bulk(ops, num_to_enq);
1354         }
1355
1356         if (allocs_failed > 0)
1357                 printf("WARNING: op allocations failed: %u times\n",
1358                                 allocs_failed);
1359
1360         return TEST_SUCCESS;
1361 }
1362
1363 static int
1364 throughput_pmd_lcore_dec(void *arg)
1365 {
1366         struct thread_params *tp = arg;
1367         unsigned int enqueued, dequeued;
1368         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1369         uint64_t total_time, start_time;
1370         const uint16_t queue_id = tp->queue_id;
1371         const uint16_t burst_sz = tp->op_params->burst_sz;
1372         const uint16_t num_to_process = tp->op_params->num_to_process;
1373         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1374         struct test_buffers *bufs = NULL;
1375         unsigned int allocs_failed = 0;
1376         int ret;
1377         struct rte_bbdev_info info;
1378
1379         /* Input length in bytes, million operations per second, million bits
1380          * per second.
1381          */
1382         double in_len;
1383
1384         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1385                         "BURST_SIZE should be <= %u", MAX_BURST);
1386
1387         rte_bbdev_info_get(tp->dev_id, &info);
1388         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1389
1390         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1391                 rte_pause();
1392
1393         start_time = rte_rdtsc_precise();
1394         for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) {
1395                 uint16_t deq;
1396
1397                 if (likely(enqueued < num_to_process)) {
1398
1399                         uint16_t num_to_enq = burst_sz;
1400
1401                         if (unlikely(num_to_process - enqueued < num_to_enq))
1402                                 num_to_enq = num_to_process - enqueued;
1403
1404                         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp,
1405                                         ops_enq, num_to_enq);
1406                         if (ret != 0) {
1407                                 allocs_failed++;
1408                                 goto do_dequeue;
1409                         }
1410
1411                         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1412                                 copy_reference_dec_op(ops_enq, num_to_enq,
1413                                                 enqueued,
1414                                                 bufs->inputs,
1415                                                 bufs->hard_outputs,
1416                                                 bufs->soft_outputs,
1417                                                 ref_op);
1418
1419                         enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id,
1420                                         queue_id, ops_enq, num_to_enq);
1421                 }
1422 do_dequeue:
1423                 deq = rte_bbdev_dequeue_dec_ops(tp->dev_id, queue_id, ops_deq,
1424                                 burst_sz);
1425                 dequeued += deq;
1426                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1427         }
1428         total_time = rte_rdtsc_precise() - start_time;
1429
1430         if (allocs_failed > 0)
1431                 printf("WARNING: op allocations failed: %u times\n",
1432                                 allocs_failed);
1433
1434         TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)",
1435                         enqueued, dequeued);
1436
1437         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1438                 ret = validate_dec_buffers(ref_op, bufs, num_to_process);
1439                 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed");
1440         }
1441
1442         in_len = ref_op->turbo_dec.input.length;
1443         tp->mops = ((double)num_to_process / 1000000.0) /
1444                         ((double)total_time / (double)rte_get_tsc_hz());
1445         tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1446                         ((double)total_time / (double)rte_get_tsc_hz());
1447
1448         return TEST_SUCCESS;
1449 }
1450
1451 static int
1452 throughput_pmd_lcore_enc(void *arg)
1453 {
1454         struct thread_params *tp = arg;
1455         unsigned int enqueued, dequeued;
1456         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1457         uint64_t total_time, start_time;
1458         const uint16_t queue_id = tp->queue_id;
1459         const uint16_t burst_sz = tp->op_params->burst_sz;
1460         const uint16_t num_to_process = tp->op_params->num_to_process;
1461         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1462         struct test_buffers *bufs = NULL;
1463         unsigned int allocs_failed = 0;
1464         int ret;
1465         struct rte_bbdev_info info;
1466
1467         /* Input length in bytes, million operations per second, million bits
1468          * per second.
1469          */
1470         double in_len;
1471
1472         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1473                         "BURST_SIZE should be <= %u", MAX_BURST);
1474
1475         rte_bbdev_info_get(tp->dev_id, &info);
1476         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1477
1478         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1479                 rte_pause();
1480
1481         start_time = rte_rdtsc_precise();
1482         for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) {
1483                 uint16_t deq;
1484
1485                 if (likely(enqueued < num_to_process)) {
1486
1487                         uint16_t num_to_enq = burst_sz;
1488
1489                         if (unlikely(num_to_process - enqueued < num_to_enq))
1490                                 num_to_enq = num_to_process - enqueued;
1491
1492                         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp,
1493                                         ops_enq, num_to_enq);
1494                         if (ret != 0) {
1495                                 allocs_failed++;
1496                                 goto do_dequeue;
1497                         }
1498
1499                         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1500                                 copy_reference_enc_op(ops_enq, num_to_enq,
1501                                                 enqueued,
1502                                                 bufs->inputs,
1503                                                 bufs->hard_outputs,
1504                                                 ref_op);
1505
1506                         enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id,
1507                                         queue_id, ops_enq, num_to_enq);
1508                 }
1509 do_dequeue:
1510                 deq = rte_bbdev_dequeue_enc_ops(tp->dev_id, queue_id, ops_deq,
1511                                 burst_sz);
1512                 dequeued += deq;
1513                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1514         }
1515         total_time = rte_rdtsc_precise() - start_time;
1516
1517         if (allocs_failed > 0)
1518                 printf("WARNING: op allocations failed: %u times\n",
1519                                 allocs_failed);
1520
1521         TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)",
1522                         enqueued, dequeued);
1523
1524         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1525                 ret = validate_enc_buffers(bufs, num_to_process);
1526                 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed");
1527         }
1528
1529         in_len = ref_op->turbo_enc.input.length;
1530
1531         tp->mops = ((double)num_to_process / 1000000.0) /
1532                         ((double)total_time / (double)rte_get_tsc_hz());
1533         tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1534                         ((double)total_time / (double)rte_get_tsc_hz());
1535
1536         return TEST_SUCCESS;
1537 }
1538 static void
1539 print_throughput(struct thread_params *t_params, unsigned int used_cores)
1540 {
1541         unsigned int lcore_id, iter = 0;
1542         double total_mops = 0, total_mbps = 0;
1543
1544         RTE_LCORE_FOREACH(lcore_id) {
1545                 if (iter++ >= used_cores)
1546                         break;
1547                 printf("\tlcore_id: %u, throughput: %.8lg MOPS, %.8lg Mbps\n",
1548                 lcore_id, t_params[lcore_id].mops, t_params[lcore_id].mbps);
1549                 total_mops += t_params[lcore_id].mops;
1550                 total_mbps += t_params[lcore_id].mbps;
1551         }
1552         printf(
1553                 "\n\tTotal stats for %u cores: throughput: %.8lg MOPS, %.8lg Mbps\n",
1554                 used_cores, total_mops, total_mbps);
1555 }
1556
1557 /*
1558  * Test function that determines how long an enqueue + dequeue of a burst
1559  * takes on available lcores.
1560  */
1561 static int
1562 throughput_test(struct active_device *ad,
1563                 struct test_op_params *op_params)
1564 {
1565         int ret;
1566         unsigned int lcore_id, used_cores = 0;
1567         struct thread_params t_params[MAX_QUEUES];
1568         struct rte_bbdev_info info;
1569         lcore_function_t *throughput_function;
1570         struct thread_params *tp;
1571         uint16_t num_lcores;
1572         const char *op_type_str;
1573
1574         rte_bbdev_info_get(ad->dev_id, &info);
1575
1576         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
1577         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
1578                         test_vector.op_type);
1579
1580         printf(
1581                 "Throughput test: dev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, int mode: %s, GHz: %lg\n",
1582                         info.dev_name, ad->nb_queues, op_params->burst_sz,
1583                         op_params->num_to_process, op_params->num_lcores,
1584                         op_type_str,
1585                         intr_enabled ? "Interrupt mode" : "PMD mode",
1586                         (double)rte_get_tsc_hz() / 1000000000.0);
1587
1588         /* Set number of lcores */
1589         num_lcores = (ad->nb_queues < (op_params->num_lcores))
1590                         ? ad->nb_queues
1591                         : op_params->num_lcores;
1592
1593         if (intr_enabled) {
1594                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1595                         throughput_function = throughput_intr_lcore_dec;
1596                 else
1597                         throughput_function = throughput_intr_lcore_enc;
1598
1599                 /* Dequeue interrupt callback registration */
1600                 ret = rte_bbdev_callback_register(ad->dev_id,
1601                                 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
1602                                 &t_params);
1603                 if (ret < 0)
1604                         return ret;
1605         } else {
1606                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1607                         throughput_function = throughput_pmd_lcore_dec;
1608                 else
1609                         throughput_function = throughput_pmd_lcore_enc;
1610         }
1611
1612         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
1613
1614         t_params[rte_lcore_id()].dev_id = ad->dev_id;
1615         t_params[rte_lcore_id()].op_params = op_params;
1616         t_params[rte_lcore_id()].queue_id =
1617                         ad->queue_ids[used_cores++];
1618
1619         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1620                 if (used_cores >= num_lcores)
1621                         break;
1622
1623                 t_params[lcore_id].dev_id = ad->dev_id;
1624                 t_params[lcore_id].op_params = op_params;
1625                 t_params[lcore_id].queue_id = ad->queue_ids[used_cores++];
1626
1627                 rte_eal_remote_launch(throughput_function, &t_params[lcore_id],
1628                                 lcore_id);
1629         }
1630
1631         rte_atomic16_set(&op_params->sync, SYNC_START);
1632         ret = throughput_function(&t_params[rte_lcore_id()]);
1633
1634         /* Master core is always used */
1635         used_cores = 1;
1636         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1637                 if (used_cores++ >= num_lcores)
1638                         break;
1639
1640                 ret |= rte_eal_wait_lcore(lcore_id);
1641         }
1642
1643         /* Return if test failed */
1644         if (ret)
1645                 return ret;
1646
1647         /* Print throughput if interrupts are disabled and test passed */
1648         if (!intr_enabled) {
1649                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1650                         print_throughput(t_params, num_lcores);
1651                 return ret;
1652         }
1653
1654         /* In interrupt TC we need to wait for the interrupt callback to deqeue
1655          * all pending operations. Skip waiting for queues which reported an
1656          * error using processing_status variable.
1657          * Wait for master lcore operations.
1658          */
1659         tp = &t_params[rte_lcore_id()];
1660         while ((rte_atomic16_read(&tp->nb_dequeued) <
1661                         op_params->num_to_process) &&
1662                         (rte_atomic16_read(&tp->processing_status) !=
1663                         TEST_FAILED))
1664                 rte_pause();
1665
1666         ret |= rte_atomic16_read(&tp->processing_status);
1667
1668         /* Wait for slave lcores operations */
1669         used_cores = 1;
1670         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1671                 tp = &t_params[lcore_id];
1672                 if (used_cores++ >= num_lcores)
1673                         break;
1674
1675                 while ((rte_atomic16_read(&tp->nb_dequeued) <
1676                                 op_params->num_to_process) &&
1677                                 (rte_atomic16_read(&tp->processing_status) !=
1678                                 TEST_FAILED))
1679                         rte_pause();
1680
1681                 ret |= rte_atomic16_read(&tp->processing_status);
1682         }
1683
1684         /* Print throughput if test passed */
1685         if (!ret && test_vector.op_type != RTE_BBDEV_OP_NONE)
1686                 print_throughput(t_params, num_lcores);
1687
1688         return ret;
1689 }
1690
1691 static int
1692 operation_latency_test_dec(struct rte_mempool *mempool,
1693                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
1694                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
1695                 const uint16_t num_to_process, uint16_t burst_sz,
1696                 uint64_t *total_time)
1697 {
1698         int ret = TEST_SUCCESS;
1699         uint16_t i, j, dequeued;
1700         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1701         uint64_t start_time = 0;
1702
1703         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1704                 uint16_t enq = 0, deq = 0;
1705                 bool first_time = true;
1706
1707                 if (unlikely(num_to_process - dequeued < burst_sz))
1708                         burst_sz = num_to_process - dequeued;
1709
1710                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1711                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1712                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1713                                         bufs->inputs,
1714                                         bufs->hard_outputs,
1715                                         bufs->soft_outputs,
1716                                         ref_op);
1717
1718                 /* Set counter to validate the ordering */
1719                 for (j = 0; j < burst_sz; ++j)
1720                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1721
1722                 start_time = rte_rdtsc_precise();
1723
1724                 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
1725                                 burst_sz);
1726                 TEST_ASSERT(enq == burst_sz,
1727                                 "Error enqueueing burst, expected %u, got %u",
1728                                 burst_sz, enq);
1729
1730                 /* Dequeue */
1731                 do {
1732                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1733                                         &ops_deq[deq], burst_sz - deq);
1734                         if (likely(first_time && (deq > 0))) {
1735                                 *total_time += rte_rdtsc_precise() - start_time;
1736                                 first_time = false;
1737                         }
1738                 } while (unlikely(burst_sz != deq));
1739
1740                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1741                         ret = validate_dec_op(ops_deq, burst_sz, ref_op,
1742                                         vector_mask);
1743                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1744                 }
1745
1746                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1747                 dequeued += deq;
1748         }
1749
1750         return i;
1751 }
1752
1753 static int
1754 operation_latency_test_enc(struct rte_mempool *mempool,
1755                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
1756                 uint16_t dev_id, uint16_t queue_id,
1757                 const uint16_t num_to_process, uint16_t burst_sz,
1758                 uint64_t *total_time)
1759 {
1760         int ret = TEST_SUCCESS;
1761         uint16_t i, j, dequeued;
1762         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1763         uint64_t start_time = 0;
1764
1765         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1766                 uint16_t enq = 0, deq = 0;
1767                 bool first_time = true;
1768
1769                 if (unlikely(num_to_process - dequeued < burst_sz))
1770                         burst_sz = num_to_process - dequeued;
1771
1772                 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
1773                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1774                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
1775                                         bufs->inputs,
1776                                         bufs->hard_outputs,
1777                                         ref_op);
1778
1779                 /* Set counter to validate the ordering */
1780                 for (j = 0; j < burst_sz; ++j)
1781                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1782
1783                 start_time = rte_rdtsc_precise();
1784
1785                 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
1786                                 burst_sz);
1787                 TEST_ASSERT(enq == burst_sz,
1788                                 "Error enqueueing burst, expected %u, got %u",
1789                                 burst_sz, enq);
1790
1791                 /* Dequeue */
1792                 do {
1793                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1794                                         &ops_deq[deq], burst_sz - deq);
1795                         if (likely(first_time && (deq > 0))) {
1796                                 *total_time += rte_rdtsc_precise() - start_time;
1797                                 first_time = false;
1798                         }
1799                 } while (unlikely(burst_sz != deq));
1800
1801                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1802                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
1803                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1804                 }
1805
1806                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1807                 dequeued += deq;
1808         }
1809
1810         return i;
1811 }
1812
1813 static int
1814 operation_latency_test(struct active_device *ad,
1815                 struct test_op_params *op_params)
1816 {
1817         int iter;
1818         uint16_t burst_sz = op_params->burst_sz;
1819         const uint16_t num_to_process = op_params->num_to_process;
1820         const enum rte_bbdev_op_type op_type = test_vector.op_type;
1821         const uint16_t queue_id = ad->queue_ids[0];
1822         struct test_buffers *bufs = NULL;
1823         struct rte_bbdev_info info;
1824         uint64_t total_time = 0;
1825         const char *op_type_str;
1826
1827         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1828                         "BURST_SIZE should be <= %u", MAX_BURST);
1829
1830         rte_bbdev_info_get(ad->dev_id, &info);
1831         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1832
1833         op_type_str = rte_bbdev_op_type_str(op_type);
1834         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
1835
1836         printf(
1837                 "Validation/Latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
1838                         info.dev_name, burst_sz, num_to_process, op_type_str);
1839
1840         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1841                 iter = operation_latency_test_dec(op_params->mp, bufs,
1842                                 op_params->ref_dec_op, op_params->vector_mask,
1843                                 ad->dev_id, queue_id, num_to_process,
1844                                 burst_sz, &total_time);
1845         else
1846                 iter = operation_latency_test_enc(op_params->mp, bufs,
1847                                 op_params->ref_enc_op, ad->dev_id, queue_id,
1848                                 num_to_process, burst_sz, &total_time);
1849
1850         if (iter <= 0)
1851                 return TEST_FAILED;
1852
1853         printf("\toperation avg. latency: %lg cycles, %lg us\n",
1854                         (double)total_time / (double)iter,
1855                         (double)(total_time * 1000000) / (double)iter /
1856                         (double)rte_get_tsc_hz());
1857
1858         return TEST_SUCCESS;
1859 }
1860
1861 static int
1862 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
1863                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
1864                 uint16_t queue_id, const uint16_t num_to_process,
1865                 uint16_t burst_sz, uint64_t *enq_total_time,
1866                 uint64_t *deq_total_time)
1867 {
1868         int i, dequeued;
1869         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1870         uint64_t enq_start_time, deq_start_time;
1871
1872         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1873                 uint16_t enq = 0, deq = 0;
1874
1875                 if (unlikely(num_to_process - dequeued < burst_sz))
1876                         burst_sz = num_to_process - dequeued;
1877
1878                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1879                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1880                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1881                                         bufs->inputs,
1882                                         bufs->hard_outputs,
1883                                         bufs->soft_outputs,
1884                                         ref_op);
1885
1886                 /* Start time measurment for enqueue function offload latency */
1887                 enq_start_time = rte_rdtsc();
1888                 do {
1889                         enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
1890                                         &ops_enq[enq], burst_sz - enq);
1891                 } while (unlikely(burst_sz != enq));
1892                 *enq_total_time += rte_rdtsc() - enq_start_time;
1893
1894                 /* ensure enqueue has been completed */
1895                 rte_delay_ms(10);
1896
1897                 /* Start time measurment for dequeue function offload latency */
1898                 deq_start_time = rte_rdtsc();
1899                 do {
1900                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1901                                         &ops_deq[deq], burst_sz - deq);
1902                 } while (unlikely(burst_sz != deq));
1903                 *deq_total_time += rte_rdtsc() - deq_start_time;
1904
1905                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1906                 dequeued += deq;
1907         }
1908
1909         return i;
1910 }
1911
1912 static int
1913 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
1914                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
1915                 uint16_t queue_id, const uint16_t num_to_process,
1916                 uint16_t burst_sz, uint64_t *enq_total_time,
1917                 uint64_t *deq_total_time)
1918 {
1919         int i, dequeued;
1920         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1921         uint64_t enq_start_time, deq_start_time;
1922
1923         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1924                 uint16_t enq = 0, deq = 0;
1925
1926                 if (unlikely(num_to_process - dequeued < burst_sz))
1927                         burst_sz = num_to_process - dequeued;
1928
1929                 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
1930                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1931                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
1932                                         bufs->inputs,
1933                                         bufs->hard_outputs,
1934                                         ref_op);
1935
1936                 /* Start time measurment for enqueue function offload latency */
1937                 enq_start_time = rte_rdtsc();
1938                 do {
1939                         enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
1940                                         &ops_enq[enq], burst_sz - enq);
1941                 } while (unlikely(burst_sz != enq));
1942                 *enq_total_time += rte_rdtsc() - enq_start_time;
1943
1944                 /* ensure enqueue has been completed */
1945                 rte_delay_ms(10);
1946
1947                 /* Start time measurment for dequeue function offload latency */
1948                 deq_start_time = rte_rdtsc();
1949                 do {
1950                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1951                                         &ops_deq[deq], burst_sz - deq);
1952                 } while (unlikely(burst_sz != deq));
1953                 *deq_total_time += rte_rdtsc() - deq_start_time;
1954
1955                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1956                 dequeued += deq;
1957         }
1958
1959         return i;
1960 }
1961
1962 static int
1963 offload_latency_test(struct active_device *ad,
1964                 struct test_op_params *op_params)
1965 {
1966         int iter;
1967         uint64_t enq_total_time = 0, deq_total_time = 0;
1968         uint16_t burst_sz = op_params->burst_sz;
1969         const uint16_t num_to_process = op_params->num_to_process;
1970         const enum rte_bbdev_op_type op_type = test_vector.op_type;
1971         const uint16_t queue_id = ad->queue_ids[0];
1972         struct test_buffers *bufs = NULL;
1973         struct rte_bbdev_info info;
1974         const char *op_type_str;
1975
1976         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1977                         "BURST_SIZE should be <= %u", MAX_BURST);
1978
1979         rte_bbdev_info_get(ad->dev_id, &info);
1980         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1981
1982         op_type_str = rte_bbdev_op_type_str(op_type);
1983         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
1984
1985         printf(
1986                 "Offload latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
1987                         info.dev_name, burst_sz, num_to_process, op_type_str);
1988
1989         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1990                 iter = offload_latency_test_dec(op_params->mp, bufs,
1991                                 op_params->ref_dec_op, ad->dev_id, queue_id,
1992                                 num_to_process, burst_sz, &enq_total_time,
1993                                 &deq_total_time);
1994         else
1995                 iter = offload_latency_test_enc(op_params->mp, bufs,
1996                                 op_params->ref_enc_op, ad->dev_id, queue_id,
1997                                 num_to_process, burst_sz, &enq_total_time,
1998                                 &deq_total_time);
1999
2000         if (iter <= 0)
2001                 return TEST_FAILED;
2002
2003         printf("\tenq offload avg. latency: %lg cycles, %lg us\n",
2004                         (double)enq_total_time / (double)iter,
2005                         (double)(enq_total_time * 1000000) / (double)iter /
2006                         (double)rte_get_tsc_hz());
2007
2008         printf("\tdeq offload avg. latency: %lg cycles, %lg us\n",
2009                         (double)deq_total_time / (double)iter,
2010                         (double)(deq_total_time * 1000000) / (double)iter /
2011                         (double)rte_get_tsc_hz());
2012
2013         return TEST_SUCCESS;
2014 }
2015
2016 static int
2017 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
2018                 const uint16_t num_to_process, uint16_t burst_sz,
2019                 uint64_t *deq_total_time)
2020 {
2021         int i, deq_total;
2022         struct rte_bbdev_dec_op *ops[MAX_BURST];
2023         uint64_t deq_start_time;
2024
2025         /* Test deq offload latency from an empty queue */
2026         deq_start_time = rte_rdtsc_precise();
2027         for (i = 0, deq_total = 0; deq_total < num_to_process;
2028                         ++i, deq_total += burst_sz) {
2029                 if (unlikely(num_to_process - deq_total < burst_sz))
2030                         burst_sz = num_to_process - deq_total;
2031                 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz);
2032         }
2033         *deq_total_time = rte_rdtsc_precise() - deq_start_time;
2034
2035         return i;
2036 }
2037
2038 static int
2039 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
2040                 const uint16_t num_to_process, uint16_t burst_sz,
2041                 uint64_t *deq_total_time)
2042 {
2043         int i, deq_total;
2044         struct rte_bbdev_enc_op *ops[MAX_BURST];
2045         uint64_t deq_start_time;
2046
2047         /* Test deq offload latency from an empty queue */
2048         deq_start_time = rte_rdtsc_precise();
2049         for (i = 0, deq_total = 0; deq_total < num_to_process;
2050                         ++i, deq_total += burst_sz) {
2051                 if (unlikely(num_to_process - deq_total < burst_sz))
2052                         burst_sz = num_to_process - deq_total;
2053                 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz);
2054         }
2055         *deq_total_time = rte_rdtsc_precise() - deq_start_time;
2056
2057         return i;
2058 }
2059
2060 static int
2061 offload_latency_empty_q_test(struct active_device *ad,
2062                 struct test_op_params *op_params)
2063 {
2064         int iter;
2065         uint64_t deq_total_time = 0;
2066         uint16_t burst_sz = op_params->burst_sz;
2067         const uint16_t num_to_process = op_params->num_to_process;
2068         const enum rte_bbdev_op_type op_type = test_vector.op_type;
2069         const uint16_t queue_id = ad->queue_ids[0];
2070         struct rte_bbdev_info info;
2071         const char *op_type_str;
2072
2073         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2074                         "BURST_SIZE should be <= %u", MAX_BURST);
2075
2076         rte_bbdev_info_get(ad->dev_id, &info);
2077
2078         op_type_str = rte_bbdev_op_type_str(op_type);
2079         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2080
2081         printf(
2082                 "Offload latency empty dequeue test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
2083                         info.dev_name, burst_sz, num_to_process, op_type_str);
2084
2085         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2086                 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
2087                                 num_to_process, burst_sz, &deq_total_time);
2088         else
2089                 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
2090                                 num_to_process, burst_sz, &deq_total_time);
2091
2092         if (iter <= 0)
2093                 return TEST_FAILED;
2094
2095         printf("\tempty deq offload avg. latency: %lg cycles, %lg us\n",
2096                         (double)deq_total_time / (double)iter,
2097                         (double)(deq_total_time * 1000000) / (double)iter /
2098                         (double)rte_get_tsc_hz());
2099
2100         return TEST_SUCCESS;
2101 }
2102
2103 static int
2104 throughput_tc(void)
2105 {
2106         return run_test_case(throughput_test);
2107 }
2108
2109 static int
2110 offload_latency_tc(void)
2111 {
2112         return run_test_case(offload_latency_test);
2113 }
2114
2115 static int
2116 offload_latency_empty_q_tc(void)
2117 {
2118         return run_test_case(offload_latency_empty_q_test);
2119 }
2120
2121 static int
2122 operation_latency_tc(void)
2123 {
2124         return run_test_case(operation_latency_test);
2125 }
2126
2127 static int
2128 interrupt_tc(void)
2129 {
2130         return run_test_case(throughput_test);
2131 }
2132
2133 static struct unit_test_suite bbdev_throughput_testsuite = {
2134         .suite_name = "BBdev Throughput Tests",
2135         .setup = testsuite_setup,
2136         .teardown = testsuite_teardown,
2137         .unit_test_cases = {
2138                 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
2139                 TEST_CASES_END() /**< NULL terminate unit test array */
2140         }
2141 };
2142
2143 static struct unit_test_suite bbdev_validation_testsuite = {
2144         .suite_name = "BBdev Validation Tests",
2145         .setup = testsuite_setup,
2146         .teardown = testsuite_teardown,
2147         .unit_test_cases = {
2148                 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc),
2149                 TEST_CASES_END() /**< NULL terminate unit test array */
2150         }
2151 };
2152
2153 static struct unit_test_suite bbdev_latency_testsuite = {
2154         .suite_name = "BBdev Latency Tests",
2155         .setup = testsuite_setup,
2156         .teardown = testsuite_teardown,
2157         .unit_test_cases = {
2158                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_tc),
2159                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
2160                 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc),
2161                 TEST_CASES_END() /**< NULL terminate unit test array */
2162         }
2163 };
2164
2165 static struct unit_test_suite bbdev_interrupt_testsuite = {
2166         .suite_name = "BBdev Interrupt Tests",
2167         .setup = interrupt_testsuite_setup,
2168         .teardown = testsuite_teardown,
2169         .unit_test_cases = {
2170                 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
2171                 TEST_CASES_END() /**< NULL terminate unit test array */
2172         }
2173 };
2174
2175 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
2176 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
2177 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
2178 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);