app/test-bbdev/test_bbdev_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7 #include <math.h>
   8
   9 #include <rte_eal.h>
  10 #include <rte_common.h>
  11 #include <rte_dev.h>
  12 #include <rte_launch.h>
  13 #include <rte_bbdev.h>
  14 #include <rte_cycles.h>
  15 #include <rte_lcore.h>
  16 #include <rte_malloc.h>
  17 #include <rte_random.h>
  18 #include <rte_hexdump.h>
  19
  20 #include "main.h"
  21 #include "test_bbdev_vector.h"
  22
  23 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
  24
  25 #define MAX_QUEUES RTE_MAX_LCORE
  26 #define TEST_REPETITIONS 1000
  27
  28 #define OPS_CACHE_SIZE 256U
  29 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
  30
  31 #define SYNC_WAIT 0
  32 #define SYNC_START 1
  33
  34 #define INVALID_QUEUE_ID -1
  35
  36 static struct test_bbdev_vector test_vector;
  37
  38 /* Switch between PMD and Interrupt for throughput TC */
  39 static bool intr_enabled;
  40
  41 /* Represents tested active devices */
  42 static struct active_device {
  43         const char *driver_name;
  44         uint8_t dev_id;
  45         uint16_t supported_ops;
  46         uint16_t queue_ids[MAX_QUEUES];
  47         uint16_t nb_queues;
  48         struct rte_mempool *ops_mempool;
  49         struct rte_mempool *in_mbuf_pool;
  50         struct rte_mempool *hard_out_mbuf_pool;
  51         struct rte_mempool *soft_out_mbuf_pool;
  52 } active_devs[RTE_BBDEV_MAX_DEVS];
  53
  54 static uint8_t nb_active_devs;
  55
  56 /* Data buffers used by BBDEV ops */
  57 struct test_buffers {
  58         struct rte_bbdev_op_data *inputs;
  59         struct rte_bbdev_op_data *hard_outputs;
  60         struct rte_bbdev_op_data *soft_outputs;
  61 };
  62
  63 /* Operation parameters specific for given test case */
  64 struct test_op_params {
  65         struct rte_mempool *mp;
  66         struct rte_bbdev_dec_op *ref_dec_op;
  67         struct rte_bbdev_enc_op *ref_enc_op;
  68         uint16_t burst_sz;
  69         uint16_t num_to_process;
  70         uint16_t num_lcores;
  71         int vector_mask;
  72         rte_atomic16_t sync;
  73         struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
  74 };
  75
  76 /* Contains per lcore params */
  77 struct thread_params {
  78         uint8_t dev_id;
  79         uint16_t queue_id;
  80         uint32_t lcore_id;
  81         uint64_t start_time;
  82         double ops_per_sec;
  83         double mbps;
  84         uint8_t iter_count;
  85         rte_atomic16_t nb_dequeued;
  86         rte_atomic16_t processing_status;
  87         rte_atomic16_t burst_sz;
  88         struct test_op_params *op_params;
  89         struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
  90         struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
  91 };
  92
  93 #ifdef RTE_BBDEV_OFFLOAD_COST
  94 /* Stores time statistics */
  95 struct test_time_stats {
  96         /* Stores software enqueue total working time */
  97         uint64_t enq_sw_total_time;
  98         /* Stores minimum value of software enqueue working time */
  99         uint64_t enq_sw_min_time;
 100         /* Stores maximum value of software enqueue working time */
 101         uint64_t enq_sw_max_time;
 102         /* Stores turbo enqueue total working time */
 103         uint64_t enq_acc_total_time;
 104         /* Stores minimum value of accelerator enqueue working time */
 105         uint64_t enq_acc_min_time;
 106         /* Stores maximum value of accelerator enqueue working time */
 107         uint64_t enq_acc_max_time;
 108         /* Stores dequeue total working time */
 109         uint64_t deq_total_time;
 110         /* Stores minimum value of dequeue working time */
 111         uint64_t deq_min_time;
 112         /* Stores maximum value of dequeue working time */
 113         uint64_t deq_max_time;
 114 };
 115 #endif
 116
 117 typedef int (test_case_function)(struct active_device *ad,
 118                 struct test_op_params *op_params);
 119
 120 static inline void
 121 mbuf_reset(struct rte_mbuf *m)
 122 {
 123         m->pkt_len = 0;
 124
 125         do {
 126                 m->data_len = 0;
 127                 m = m->next;
 128         } while (m != NULL);
 129 }
 130
 131 static inline void
 132 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 133 {
 134         ad->supported_ops |= (1 << op_type);
 135 }
 136
 137 static inline bool
 138 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 139 {
 140         return ad->supported_ops & (1 << op_type);
 141 }
 142
 143 static inline bool
 144 flags_match(uint32_t flags_req, uint32_t flags_present)
 145 {
 146         return (flags_req & flags_present) == flags_req;
 147 }
 148
 149 static void
 150 clear_soft_out_cap(uint32_t *op_flags)
 151 {
 152         *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
 153         *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
 154         *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
 155 }
 156
 157 static int
 158 check_dev_cap(const struct rte_bbdev_info *dev_info)
 159 {
 160         unsigned int i;
 161         unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs;
 162         const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
 163
 164         nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
 165         nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
 166         nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
 167
 168         for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
 169                 if (op_cap->type != test_vector.op_type)
 170                         continue;
 171
 172                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
 173                         const struct rte_bbdev_op_cap_turbo_dec *cap =
 174                                         &op_cap->cap.turbo_dec;
 175                         /* Ignore lack of soft output capability, just skip
 176                          * checking if soft output is valid.
 177                          */
 178                         if ((test_vector.turbo_dec.op_flags &
 179                                         RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
 180                                         !(cap->capability_flags &
 181                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 182                                 printf(
 183                                         "WARNING: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
 184                                         dev_info->dev_name);
 185                                 clear_soft_out_cap(
 186                                         &test_vector.turbo_dec.op_flags);
 187                         }
 188
 189                         if (!flags_match(test_vector.turbo_dec.op_flags,
 190                                         cap->capability_flags))
 191                                 return TEST_FAILED;
 192                         if (nb_inputs > cap->num_buffers_src) {
 193                                 printf("Too many inputs defined: %u, max: %u\n",
 194                                         nb_inputs, cap->num_buffers_src);
 195                                 return TEST_FAILED;
 196                         }
 197                         if (nb_soft_outputs > cap->num_buffers_soft_out &&
 198                                         (test_vector.turbo_dec.op_flags &
 199                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 200                                 printf(
 201                                         "Too many soft outputs defined: %u, max: %u\n",
 202                                                 nb_soft_outputs,
 203                                                 cap->num_buffers_soft_out);
 204                                 return TEST_FAILED;
 205                         }
 206                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 207                                 printf(
 208                                         "Too many hard outputs defined: %u, max: %u\n",
 209                                                 nb_hard_outputs,
 210                                                 cap->num_buffers_hard_out);
 211                                 return TEST_FAILED;
 212                         }
 213                         if (intr_enabled && !(cap->capability_flags &
 214                                         RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
 215                                 printf(
 216                                         "Dequeue interrupts are not supported!\n");
 217                                 return TEST_FAILED;
 218                         }
 219
 220                         return TEST_SUCCESS;
 221                 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
 222                         const struct rte_bbdev_op_cap_turbo_enc *cap =
 223                                         &op_cap->cap.turbo_enc;
 224
 225                         if (!flags_match(test_vector.turbo_enc.op_flags,
 226                                         cap->capability_flags))
 227                                 return TEST_FAILED;
 228                         if (nb_inputs > cap->num_buffers_src) {
 229                                 printf("Too many inputs defined: %u, max: %u\n",
 230                                         nb_inputs, cap->num_buffers_src);
 231                                 return TEST_FAILED;
 232                         }
 233                         if (nb_hard_outputs > cap->num_buffers_dst) {
 234                                 printf(
 235                                         "Too many hard outputs defined: %u, max: %u\n",
 236                                         nb_hard_outputs, cap->num_buffers_src);
 237                                 return TEST_FAILED;
 238                         }
 239                         if (intr_enabled && !(cap->capability_flags &
 240                                         RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
 241                                 printf(
 242                                         "Dequeue interrupts are not supported!\n");
 243                                 return TEST_FAILED;
 244                         }
 245
 246                         return TEST_SUCCESS;
 247                 }
 248         }
 249
 250         if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
 251                 return TEST_SUCCESS; /* Special case for NULL device */
 252
 253         return TEST_FAILED;
 254 }
 255
 256 /* calculates optimal mempool size not smaller than the val */
 257 static unsigned int
 258 optimal_mempool_size(unsigned int val)
 259 {
 260         return rte_align32pow2(val + 1) - 1;
 261 }
 262
 263 /* allocates mbuf mempool for inputs and outputs */
 264 static struct rte_mempool *
 265 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
 266                 int socket_id, unsigned int mbuf_pool_size,
 267                 const char *op_type_str)
 268 {
 269         unsigned int i;
 270         uint32_t max_seg_sz = 0;
 271         char pool_name[RTE_MEMPOOL_NAMESIZE];
 272
 273         /* find max input segment size */
 274         for (i = 0; i < entries->nb_segments; ++i)
 275                 if (entries->segments[i].length > max_seg_sz)
 276                         max_seg_sz = entries->segments[i].length;
 277
 278         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 279                         dev_id);
 280         return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
 281                         RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM,
 282                         (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
 283 }
 284
 285 static int
 286 create_mempools(struct active_device *ad, int socket_id,
 287                 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
 288 {
 289         struct rte_mempool *mp;
 290         unsigned int ops_pool_size, mbuf_pool_size = 0;
 291         char pool_name[RTE_MEMPOOL_NAMESIZE];
 292         const char *op_type_str;
 293         enum rte_bbdev_op_type op_type = org_op_type;
 294
 295         struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
 296         struct op_data_entries *hard_out =
 297                         &test_vector.entries[DATA_HARD_OUTPUT];
 298         struct op_data_entries *soft_out =
 299                         &test_vector.entries[DATA_SOFT_OUTPUT];
 300
 301         /* allocate ops mempool */
 302         ops_pool_size = optimal_mempool_size(RTE_MAX(
 303                         /* Ops used plus 1 reference op */
 304                         RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
 305                         /* Minimal cache size plus 1 reference op */
 306                         (unsigned int)(1.5 * rte_lcore_count() *
 307                                         OPS_CACHE_SIZE + 1)),
 308                         OPS_POOL_SIZE_MIN));
 309
 310         if (org_op_type == RTE_BBDEV_OP_NONE)
 311                 op_type = RTE_BBDEV_OP_TURBO_ENC;
 312
 313         op_type_str = rte_bbdev_op_type_str(op_type);
 314         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
 315
 316         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 317                         ad->dev_id);
 318         mp = rte_bbdev_op_pool_create(pool_name, op_type,
 319                         ops_pool_size, OPS_CACHE_SIZE, socket_id);
 320         TEST_ASSERT_NOT_NULL(mp,
 321                         "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
 322                         ops_pool_size,
 323                         ad->dev_id,
 324                         socket_id);
 325         ad->ops_mempool = mp;
 326
 327         /* Do not create inputs and outputs mbufs for BaseBand Null Device */
 328         if (org_op_type == RTE_BBDEV_OP_NONE)
 329                 return TEST_SUCCESS;
 330
 331         /* Inputs */
 332         mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments);
 333         mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in");
 334         TEST_ASSERT_NOT_NULL(mp,
 335                         "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
 336                         mbuf_pool_size,
 337                         ad->dev_id,
 338                         socket_id);
 339         ad->in_mbuf_pool = mp;
 340
 341         /* Hard outputs */
 342         mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 343                         hard_out->nb_segments);
 344         mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size,
 345                         "hard_out");
 346         TEST_ASSERT_NOT_NULL(mp,
 347                         "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
 348                         mbuf_pool_size,
 349                         ad->dev_id,
 350                         socket_id);
 351         ad->hard_out_mbuf_pool = mp;
 352
 353         if (soft_out->nb_segments == 0)
 354                 return TEST_SUCCESS;
 355
 356         /* Soft outputs */
 357         mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 358                         soft_out->nb_segments);
 359         mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, mbuf_pool_size,
 360                         "soft_out");
 361         TEST_ASSERT_NOT_NULL(mp,
 362                         "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
 363                         mbuf_pool_size,
 364                         ad->dev_id,
 365                         socket_id);
 366         ad->soft_out_mbuf_pool = mp;
 367
 368         return 0;
 369 }
 370
 371 static int
 372 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
 373                 struct test_bbdev_vector *vector)
 374 {
 375         int ret;
 376         unsigned int queue_id;
 377         struct rte_bbdev_queue_conf qconf;
 378         struct active_device *ad = &active_devs[nb_active_devs];
 379         unsigned int nb_queues;
 380         enum rte_bbdev_op_type op_type = vector->op_type;
 381
 382         nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
 383         /* setup device */
 384         ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
 385         if (ret < 0) {
 386                 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
 387                                 dev_id, nb_queues, info->socket_id, ret);
 388                 return TEST_FAILED;
 389         }
 390
 391         /* configure interrupts if needed */
 392         if (intr_enabled) {
 393                 ret = rte_bbdev_intr_enable(dev_id);
 394                 if (ret < 0) {
 395                         printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
 396                                         ret);
 397                         return TEST_FAILED;
 398                 }
 399         }
 400
 401         /* setup device queues */
 402         qconf.socket = info->socket_id;
 403         qconf.queue_size = info->drv.default_queue_conf.queue_size;
 404         qconf.priority = 0;
 405         qconf.deferred_start = 0;
 406         qconf.op_type = op_type;
 407
 408         for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
 409                 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
 410                 if (ret != 0) {
 411                         printf(
 412                                         "Allocated all queues (id=%u) at prio%u on dev%u\n",
 413                                         queue_id, qconf.priority, dev_id);
 414                         qconf.priority++;
 415                         ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
 416                                         &qconf);
 417                 }
 418                 if (ret != 0) {
 419                         printf("All queues on dev %u allocated: %u\n",
 420                                         dev_id, queue_id);
 421                         break;
 422                 }
 423                 ad->queue_ids[queue_id] = queue_id;
 424         }
 425         TEST_ASSERT(queue_id != 0,
 426                         "ERROR Failed to configure any queues on dev %u",
 427                         dev_id);
 428         ad->nb_queues = queue_id;
 429
 430         set_avail_op(ad, op_type);
 431
 432         return TEST_SUCCESS;
 433 }
 434
 435 static int
 436 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
 437                 struct test_bbdev_vector *vector)
 438 {
 439         int ret;
 440
 441         active_devs[nb_active_devs].driver_name = info->drv.driver_name;
 442         active_devs[nb_active_devs].dev_id = dev_id;
 443
 444         ret = add_bbdev_dev(dev_id, info, vector);
 445         if (ret == TEST_SUCCESS)
 446                 ++nb_active_devs;
 447         return ret;
 448 }
 449
 450 static uint8_t
 451 populate_active_devices(void)
 452 {
 453         int ret;
 454         uint8_t dev_id;
 455         uint8_t nb_devs_added = 0;
 456         struct rte_bbdev_info info;
 457
 458         RTE_BBDEV_FOREACH(dev_id) {
 459                 rte_bbdev_info_get(dev_id, &info);
 460
 461                 if (check_dev_cap(&info)) {
 462                         printf(
 463                                 "Device %d (%s) does not support specified capabilities\n",
 464                                         dev_id, info.dev_name);
 465                         continue;
 466                 }
 467
 468                 ret = add_active_device(dev_id, &info, &test_vector);
 469                 if (ret != 0) {
 470                         printf("Adding active bbdev %s skipped\n",
 471                                         info.dev_name);
 472                         continue;
 473                 }
 474                 nb_devs_added++;
 475         }
 476
 477         return nb_devs_added;
 478 }
 479
 480 static int
 481 read_test_vector(void)
 482 {
 483         int ret;
 484
 485         memset(&test_vector, 0, sizeof(test_vector));
 486         printf("Test vector file = %s\n", get_vector_filename());
 487         ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
 488         TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
 489                         get_vector_filename());
 490
 491         return TEST_SUCCESS;
 492 }
 493
 494 static int
 495 testsuite_setup(void)
 496 {
 497         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 498
 499         if (populate_active_devices() == 0) {
 500                 printf("No suitable devices found!\n");
 501                 return TEST_SKIPPED;
 502         }
 503
 504         return TEST_SUCCESS;
 505 }
 506
 507 static int
 508 interrupt_testsuite_setup(void)
 509 {
 510         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 511
 512         /* Enable interrupts */
 513         intr_enabled = true;
 514
 515         /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
 516         if (populate_active_devices() == 0 ||
 517                         test_vector.op_type == RTE_BBDEV_OP_NONE) {
 518                 intr_enabled = false;
 519                 printf("No suitable devices found!\n");
 520                 return TEST_SKIPPED;
 521         }
 522
 523         return TEST_SUCCESS;
 524 }
 525
 526 static void
 527 testsuite_teardown(void)
 528 {
 529         uint8_t dev_id;
 530
 531         /* Unconfigure devices */
 532         RTE_BBDEV_FOREACH(dev_id)
 533                 rte_bbdev_close(dev_id);
 534
 535         /* Clear active devices structs. */
 536         memset(active_devs, 0, sizeof(active_devs));
 537         nb_active_devs = 0;
 538 }
 539
 540 static int
 541 ut_setup(void)
 542 {
 543         uint8_t i, dev_id;
 544
 545         for (i = 0; i < nb_active_devs; i++) {
 546                 dev_id = active_devs[i].dev_id;
 547                 /* reset bbdev stats */
 548                 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
 549                                 "Failed to reset stats of bbdev %u", dev_id);
 550                 /* start the device */
 551                 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
 552                                 "Failed to start bbdev %u", dev_id);
 553         }
 554
 555         return TEST_SUCCESS;
 556 }
 557
 558 static void
 559 ut_teardown(void)
 560 {
 561         uint8_t i, dev_id;
 562         struct rte_bbdev_stats stats;
 563
 564         for (i = 0; i < nb_active_devs; i++) {
 565                 dev_id = active_devs[i].dev_id;
 566                 /* read stats and print */
 567                 rte_bbdev_stats_get(dev_id, &stats);
 568                 /* Stop the device */
 569                 rte_bbdev_stop(dev_id);
 570         }
 571 }
 572
 573 static int
 574 init_op_data_objs(struct rte_bbdev_op_data *bufs,
 575                 struct op_data_entries *ref_entries,
 576                 struct rte_mempool *mbuf_pool, const uint16_t n,
 577                 enum op_data_type op_type, uint16_t min_alignment)
 578 {
 579         int ret;
 580         unsigned int i, j;
 581
 582         for (i = 0; i < n; ++i) {
 583                 char *data;
 584                 struct op_data_buf *seg = &ref_entries->segments[0];
 585                 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
 586                 TEST_ASSERT_NOT_NULL(m_head,
 587                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 588                                 op_type, n * ref_entries->nb_segments,
 589                                 mbuf_pool->size);
 590
 591                 TEST_ASSERT_SUCCESS(((seg->length + RTE_PKTMBUF_HEADROOM) >
 592                                 (uint32_t)UINT16_MAX),
 593                                 "Given data is bigger than allowed mbuf segment size");
 594
 595                 bufs[i].data = m_head;
 596                 bufs[i].offset = 0;
 597                 bufs[i].length = 0;
 598
 599                 if (op_type == DATA_INPUT) {
 600                         data = rte_pktmbuf_append(m_head, seg->length);
 601                         TEST_ASSERT_NOT_NULL(data,
 602                                         "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 603                                         seg->length, op_type);
 604
 605                         TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment),
 606                                         "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 607                                         data, min_alignment);
 608                         rte_memcpy(data, seg->addr, seg->length);
 609                         bufs[i].length += seg->length;
 610
 611                         for (j = 1; j < ref_entries->nb_segments; ++j) {
 612                                 struct rte_mbuf *m_tail =
 613                                                 rte_pktmbuf_alloc(mbuf_pool);
 614                                 TEST_ASSERT_NOT_NULL(m_tail,
 615                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 616                                                 op_type,
 617                                                 n * ref_entries->nb_segments,
 618                                                 mbuf_pool->size);
 619                                 seg += 1;
 620
 621                                 data = rte_pktmbuf_append(m_tail, seg->length);
 622                                 TEST_ASSERT_NOT_NULL(data,
 623                                                 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 624                                                 seg->length, op_type);
 625
 626                                 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
 627                                                 min_alignment),
 628                                                 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 629                                                 data, min_alignment);
 630                                 rte_memcpy(data, seg->addr, seg->length);
 631                                 bufs[i].length += seg->length;
 632
 633                                 ret = rte_pktmbuf_chain(m_head, m_tail);
 634                                 TEST_ASSERT_SUCCESS(ret,
 635                                                 "Couldn't chain mbufs from %d data type mbuf pool",
 636                                                 op_type);
 637                         }
 638
 639                 } else {
 640
 641                         /* allocate chained-mbuf for output buffer */
 642                         for (j = 1; j < ref_entries->nb_segments; ++j) {
 643                                 struct rte_mbuf *m_tail =
 644                                                 rte_pktmbuf_alloc(mbuf_pool);
 645                                 TEST_ASSERT_NOT_NULL(m_tail,
 646                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 647                                                 op_type,
 648                                                 n * ref_entries->nb_segments,
 649                                                 mbuf_pool->size);
 650
 651                                 ret = rte_pktmbuf_chain(m_head, m_tail);
 652                                 TEST_ASSERT_SUCCESS(ret,
 653                                                 "Couldn't chain mbufs from %d data type mbuf pool",
 654                                                 op_type);
 655                         }
 656                 }
 657         }
 658
 659         return 0;
 660 }
 661
 662 static int
 663 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
 664                 const int socket)
 665 {
 666         int i;
 667
 668         *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
 669         if (*buffers == NULL) {
 670                 printf("WARNING: Failed to allocate op_data on socket %d\n",
 671                                 socket);
 672                 /* try to allocate memory on other detected sockets */
 673                 for (i = 0; i < socket; i++) {
 674                         *buffers = rte_zmalloc_socket(NULL, len, 0, i);
 675                         if (*buffers != NULL)
 676                                 break;
 677                 }
 678         }
 679
 680         return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
 681 }
 682
 683 static void
 684 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
 685                 uint16_t n, int8_t max_llr_modulus)
 686 {
 687         uint16_t i, byte_idx;
 688
 689         for (i = 0; i < n; ++i) {
 690                 struct rte_mbuf *m = input_ops[i].data;
 691                 while (m != NULL) {
 692                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
 693                                         input_ops[i].offset);
 694                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
 695                                         ++byte_idx)
 696                                 llr[byte_idx] = round((double)max_llr_modulus *
 697                                                 llr[byte_idx] / INT8_MAX);
 698
 699                         m = m->next;
 700                 }
 701         }
 702 }
 703
 704 static int
 705 fill_queue_buffers(struct test_op_params *op_params,
 706                 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
 707                 struct rte_mempool *soft_out_mp, uint16_t queue_id,
 708                 const struct rte_bbdev_op_cap *capabilities,
 709                 uint16_t min_alignment, const int socket_id)
 710 {
 711         int ret;
 712         enum op_data_type type;
 713         const uint16_t n = op_params->num_to_process;
 714
 715         struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
 716                 in_mp,
 717                 soft_out_mp,
 718                 hard_out_mp,
 719         };
 720
 721         struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
 722                 &op_params->q_bufs[socket_id][queue_id].inputs,
 723                 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
 724                 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
 725         };
 726
 727         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
 728                 struct op_data_entries *ref_entries =
 729                                 &test_vector.entries[type];
 730                 if (ref_entries->nb_segments == 0)
 731                         continue;
 732
 733                 ret = allocate_buffers_on_socket(queue_ops[type],
 734                                 n * sizeof(struct rte_bbdev_op_data),
 735                                 socket_id);
 736                 TEST_ASSERT_SUCCESS(ret,
 737                                 "Couldn't allocate memory for rte_bbdev_op_data structs");
 738
 739                 ret = init_op_data_objs(*queue_ops[type], ref_entries,
 740                                 mbuf_pools[type], n, type, min_alignment);
 741                 TEST_ASSERT_SUCCESS(ret,
 742                                 "Couldn't init rte_bbdev_op_data structs");
 743         }
 744
 745         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
 746                 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
 747                         capabilities->cap.turbo_dec.max_llr_modulus);
 748
 749         return 0;
 750 }
 751
 752 static void
 753 free_buffers(struct active_device *ad, struct test_op_params *op_params)
 754 {
 755         unsigned int i, j;
 756
 757         rte_mempool_free(ad->ops_mempool);
 758         rte_mempool_free(ad->in_mbuf_pool);
 759         rte_mempool_free(ad->hard_out_mbuf_pool);
 760         rte_mempool_free(ad->soft_out_mbuf_pool);
 761
 762         for (i = 0; i < rte_lcore_count(); ++i) {
 763                 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
 764                         rte_free(op_params->q_bufs[j][i].inputs);
 765                         rte_free(op_params->q_bufs[j][i].hard_outputs);
 766                         rte_free(op_params->q_bufs[j][i].soft_outputs);
 767                 }
 768         }
 769 }
 770
 771 static void
 772 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
 773                 unsigned int start_idx,
 774                 struct rte_bbdev_op_data *inputs,
 775                 struct rte_bbdev_op_data *hard_outputs,
 776                 struct rte_bbdev_op_data *soft_outputs,
 777                 struct rte_bbdev_dec_op *ref_op)
 778 {
 779         unsigned int i;
 780         struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
 781
 782         for (i = 0; i < n; ++i) {
 783                 if (turbo_dec->code_block_mode == 0) {
 784                         ops[i]->turbo_dec.tb_params.ea =
 785                                         turbo_dec->tb_params.ea;
 786                         ops[i]->turbo_dec.tb_params.eb =
 787                                         turbo_dec->tb_params.eb;
 788                         ops[i]->turbo_dec.tb_params.k_pos =
 789                                         turbo_dec->tb_params.k_pos;
 790                         ops[i]->turbo_dec.tb_params.k_neg =
 791                                         turbo_dec->tb_params.k_neg;
 792                         ops[i]->turbo_dec.tb_params.c =
 793                                         turbo_dec->tb_params.c;
 794                         ops[i]->turbo_dec.tb_params.c_neg =
 795                                         turbo_dec->tb_params.c_neg;
 796                         ops[i]->turbo_dec.tb_params.cab =
 797                                         turbo_dec->tb_params.cab;
 798                         ops[i]->turbo_dec.tb_params.r =
 799                                         turbo_dec->tb_params.r;
 800                 } else {
 801                         ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
 802                         ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
 803                 }
 804
 805                 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
 806                 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
 807                 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
 808                 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
 809                 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
 810                 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
 811                 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
 812
 813                 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
 814                 ops[i]->turbo_dec.input = inputs[start_idx + i];
 815                 if (soft_outputs != NULL)
 816                         ops[i]->turbo_dec.soft_output =
 817                                 soft_outputs[start_idx + i];
 818         }
 819 }
 820
 821 static void
 822 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
 823                 unsigned int start_idx,
 824                 struct rte_bbdev_op_data *inputs,
 825                 struct rte_bbdev_op_data *outputs,
 826                 struct rte_bbdev_enc_op *ref_op)
 827 {
 828         unsigned int i;
 829         struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
 830         for (i = 0; i < n; ++i) {
 831                 if (turbo_enc->code_block_mode == 0) {
 832                         ops[i]->turbo_enc.tb_params.ea =
 833                                         turbo_enc->tb_params.ea;
 834                         ops[i]->turbo_enc.tb_params.eb =
 835                                         turbo_enc->tb_params.eb;
 836                         ops[i]->turbo_enc.tb_params.k_pos =
 837                                         turbo_enc->tb_params.k_pos;
 838                         ops[i]->turbo_enc.tb_params.k_neg =
 839                                         turbo_enc->tb_params.k_neg;
 840                         ops[i]->turbo_enc.tb_params.c =
 841                                         turbo_enc->tb_params.c;
 842                         ops[i]->turbo_enc.tb_params.c_neg =
 843                                         turbo_enc->tb_params.c_neg;
 844                         ops[i]->turbo_enc.tb_params.cab =
 845                                         turbo_enc->tb_params.cab;
 846                         ops[i]->turbo_enc.tb_params.ncb_pos =
 847                                         turbo_enc->tb_params.ncb_pos;
 848                         ops[i]->turbo_enc.tb_params.ncb_neg =
 849                                         turbo_enc->tb_params.ncb_neg;
 850                         ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
 851                 } else {
 852                         ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
 853                         ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
 854                         ops[i]->turbo_enc.cb_params.ncb =
 855                                         turbo_enc->cb_params.ncb;
 856                 }
 857                 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
 858                 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
 859                 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
 860
 861                 ops[i]->turbo_enc.output = outputs[start_idx + i];
 862                 ops[i]->turbo_enc.input = inputs[start_idx + i];
 863         }
 864 }
 865
 866 static int
 867 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
 868                 unsigned int order_idx, const int expected_status)
 869 {
 870         TEST_ASSERT(op->status == expected_status,
 871                         "op_status (%d) != expected_status (%d)",
 872                         op->status, expected_status);
 873
 874         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
 875                         "Ordering error, expected %p, got %p",
 876                         (void *)(uintptr_t)order_idx, op->opaque_data);
 877
 878         return TEST_SUCCESS;
 879 }
 880
 881 static int
 882 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
 883                 unsigned int order_idx, const int expected_status)
 884 {
 885         TEST_ASSERT(op->status == expected_status,
 886                         "op_status (%d) != expected_status (%d)",
 887                         op->status, expected_status);
 888
 889         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
 890                         "Ordering error, expected %p, got %p",
 891                         (void *)(uintptr_t)order_idx, op->opaque_data);
 892
 893         return TEST_SUCCESS;
 894 }
 895
 896 static inline int
 897 validate_op_chain(struct rte_bbdev_op_data *op,
 898                 struct op_data_entries *orig_op)
 899 {
 900         uint8_t i;
 901         struct rte_mbuf *m = op->data;
 902         uint8_t nb_dst_segments = orig_op->nb_segments;
 903         uint32_t total_data_size = 0;
 904
 905         TEST_ASSERT(nb_dst_segments == m->nb_segs,
 906                         "Number of segments differ in original (%u) and filled (%u) op",
 907                         nb_dst_segments, m->nb_segs);
 908
 909         /* Validate each mbuf segment length */
 910         for (i = 0; i < nb_dst_segments; ++i) {
 911                 /* Apply offset to the first mbuf segment */
 912                 uint16_t offset = (i == 0) ? op->offset : 0;
 913                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
 914                 total_data_size += orig_op->segments[i].length;
 915
 916                 TEST_ASSERT(orig_op->segments[i].length == data_len,
 917                                 "Length of segment differ in original (%u) and filled (%u) op",
 918                                 orig_op->segments[i].length, data_len);
 919                 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
 920                                 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
 921                                 data_len,
 922                                 "Output buffers (CB=%u) are not equal", i);
 923                 m = m->next;
 924         }
 925
 926         /* Validate total mbuf pkt length */
 927         uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
 928         TEST_ASSERT(total_data_size == pkt_len,
 929                         "Length of data differ in original (%u) and filled (%u) op",
 930                         total_data_size, pkt_len);
 931
 932         return TEST_SUCCESS;
 933 }
 934
 935 static int
 936 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
 937                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
 938 {
 939         unsigned int i;
 940         int ret;
 941         struct op_data_entries *hard_data_orig =
 942                         &test_vector.entries[DATA_HARD_OUTPUT];
 943         struct op_data_entries *soft_data_orig =
 944                         &test_vector.entries[DATA_SOFT_OUTPUT];
 945         struct rte_bbdev_op_turbo_dec *ops_td;
 946         struct rte_bbdev_op_data *hard_output;
 947         struct rte_bbdev_op_data *soft_output;
 948         struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
 949
 950         for (i = 0; i < n; ++i) {
 951                 ops_td = &ops[i]->turbo_dec;
 952                 hard_output = &ops_td->hard_output;
 953                 soft_output = &ops_td->soft_output;
 954
 955                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
 956                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
 957                                         "Returned iter_count (%d) > expected iter_count (%d)",
 958                                         ops_td->iter_count, ref_td->iter_count);
 959                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
 960                 TEST_ASSERT_SUCCESS(ret,
 961                                 "Checking status and ordering for decoder failed");
 962
 963                 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
 964                                 hard_data_orig),
 965                                 "Hard output buffers (CB=%u) are not equal",
 966                                 i);
 967
 968                 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
 969                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
 970                                         soft_data_orig),
 971                                         "Soft output buffers (CB=%u) are not equal",
 972                                         i);
 973         }
 974
 975         return TEST_SUCCESS;
 976 }
 977
 978 static int
 979 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
 980                 struct rte_bbdev_enc_op *ref_op)
 981 {
 982         unsigned int i;
 983         int ret;
 984         struct op_data_entries *hard_data_orig =
 985                         &test_vector.entries[DATA_HARD_OUTPUT];
 986
 987         for (i = 0; i < n; ++i) {
 988                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
 989                 TEST_ASSERT_SUCCESS(ret,
 990                                 "Checking status and ordering for encoder failed");
 991                 TEST_ASSERT_SUCCESS(validate_op_chain(
 992                                 &ops[i]->turbo_enc.output,
 993                                 hard_data_orig),
 994                                 "Output buffers (CB=%u) are not equal",
 995                                 i);
 996         }
 997
 998         return TEST_SUCCESS;
 999 }
1000
1001 static void
1002 create_reference_dec_op(struct rte_bbdev_dec_op *op)
1003 {
1004         unsigned int i;
1005         struct op_data_entries *entry;
1006
1007         op->turbo_dec = test_vector.turbo_dec;
1008         entry = &test_vector.entries[DATA_INPUT];
1009         for (i = 0; i < entry->nb_segments; ++i)
1010                 op->turbo_dec.input.length +=
1011                                 entry->segments[i].length;
1012 }
1013
1014 static void
1015 create_reference_enc_op(struct rte_bbdev_enc_op *op)
1016 {
1017         unsigned int i;
1018         struct op_data_entries *entry;
1019
1020         op->turbo_enc = test_vector.turbo_enc;
1021         entry = &test_vector.entries[DATA_INPUT];
1022         for (i = 0; i < entry->nb_segments; ++i)
1023                 op->turbo_enc.input.length +=
1024                                 entry->segments[i].length;
1025 }
1026
1027 static uint32_t
1028 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
1029 {
1030         uint8_t i;
1031         uint32_t c, r, tb_size = 0;
1032
1033         if (op->turbo_dec.code_block_mode) {
1034                 tb_size = op->turbo_dec.tb_params.k_neg;
1035         } else {
1036                 c = op->turbo_dec.tb_params.c;
1037                 r = op->turbo_dec.tb_params.r;
1038                 for (i = 0; i < c-r; i++)
1039                         tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
1040                                 op->turbo_dec.tb_params.k_neg :
1041                                 op->turbo_dec.tb_params.k_pos;
1042         }
1043         return tb_size;
1044 }
1045
1046 static uint32_t
1047 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
1048 {
1049         uint8_t i;
1050         uint32_t c, r, tb_size = 0;
1051
1052         if (op->turbo_enc.code_block_mode) {
1053                 tb_size = op->turbo_enc.tb_params.k_neg;
1054         } else {
1055                 c = op->turbo_enc.tb_params.c;
1056                 r = op->turbo_enc.tb_params.r;
1057                 for (i = 0; i < c-r; i++)
1058                         tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
1059                                 op->turbo_enc.tb_params.k_neg :
1060                                 op->turbo_enc.tb_params.k_pos;
1061         }
1062         return tb_size;
1063 }
1064
1065 static int
1066 init_test_op_params(struct test_op_params *op_params,
1067                 enum rte_bbdev_op_type op_type, const int expected_status,
1068                 const int vector_mask, struct rte_mempool *ops_mp,
1069                 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
1070 {
1071         int ret = 0;
1072         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1073                 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
1074                                 &op_params->ref_dec_op, 1);
1075         else
1076                 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
1077                                 &op_params->ref_enc_op, 1);
1078
1079         TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
1080
1081         op_params->mp = ops_mp;
1082         op_params->burst_sz = burst_sz;
1083         op_params->num_to_process = num_to_process;
1084         op_params->num_lcores = num_lcores;
1085         op_params->vector_mask = vector_mask;
1086         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1087                 op_params->ref_dec_op->status = expected_status;
1088         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
1089                 op_params->ref_enc_op->status = expected_status;
1090
1091         return 0;
1092 }
1093
1094 static int
1095 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
1096                 struct test_op_params *op_params)
1097 {
1098         int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
1099         unsigned int i;
1100         struct active_device *ad;
1101         unsigned int burst_sz = get_burst_sz();
1102         enum rte_bbdev_op_type op_type = test_vector.op_type;
1103         const struct rte_bbdev_op_cap *capabilities = NULL;
1104
1105         ad = &active_devs[dev_id];
1106
1107         /* Check if device supports op_type */
1108         if (!is_avail_op(ad, test_vector.op_type))
1109                 return TEST_SUCCESS;
1110
1111         struct rte_bbdev_info info;
1112         rte_bbdev_info_get(ad->dev_id, &info);
1113         socket_id = GET_SOCKET(info.socket_id);
1114
1115         f_ret = create_mempools(ad, socket_id, op_type,
1116                         get_num_ops());
1117         if (f_ret != TEST_SUCCESS) {
1118                 printf("Couldn't create mempools");
1119                 goto fail;
1120         }
1121         if (op_type == RTE_BBDEV_OP_NONE)
1122                 op_type = RTE_BBDEV_OP_TURBO_ENC;
1123
1124         f_ret = init_test_op_params(op_params, test_vector.op_type,
1125                         test_vector.expected_status,
1126                         test_vector.mask,
1127                         ad->ops_mempool,
1128                         burst_sz,
1129                         get_num_ops(),
1130                         get_num_lcores());
1131         if (f_ret != TEST_SUCCESS) {
1132                 printf("Couldn't init test op params");
1133                 goto fail;
1134         }
1135
1136         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
1137                 /* Find Decoder capabilities */
1138                 const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
1139                 while (cap->type != RTE_BBDEV_OP_NONE) {
1140                         if (cap->type == RTE_BBDEV_OP_TURBO_DEC) {
1141                                 capabilities = cap;
1142                                 break;
1143                         }
1144                 }
1145                 TEST_ASSERT_NOT_NULL(capabilities,
1146                                 "Couldn't find Decoder capabilities");
1147
1148                 create_reference_dec_op(op_params->ref_dec_op);
1149         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1150                 create_reference_enc_op(op_params->ref_enc_op);
1151
1152         for (i = 0; i < ad->nb_queues; ++i) {
1153                 f_ret = fill_queue_buffers(op_params,
1154                                 ad->in_mbuf_pool,
1155                                 ad->hard_out_mbuf_pool,
1156                                 ad->soft_out_mbuf_pool,
1157                                 ad->queue_ids[i],
1158                                 capabilities,
1159                                 info.drv.min_alignment,
1160                                 socket_id);
1161                 if (f_ret != TEST_SUCCESS) {
1162                         printf("Couldn't init queue buffers");
1163                         goto fail;
1164                 }
1165         }
1166
1167         /* Run test case function */
1168         t_ret = test_case_func(ad, op_params);
1169
1170         /* Free active device resources and return */
1171         free_buffers(ad, op_params);
1172         return t_ret;
1173
1174 fail:
1175         free_buffers(ad, op_params);
1176         return TEST_FAILED;
1177 }
1178
1179 /* Run given test function per active device per supported op type
1180  * per burst size.
1181  */
1182 static int
1183 run_test_case(test_case_function *test_case_func)
1184 {
1185         int ret = 0;
1186         uint8_t dev;
1187
1188         /* Alloc op_params */
1189         struct test_op_params *op_params = rte_zmalloc(NULL,
1190                         sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
1191         TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
1192                         RTE_ALIGN(sizeof(struct test_op_params),
1193                                 RTE_CACHE_LINE_SIZE));
1194
1195         /* For each device run test case function */
1196         for (dev = 0; dev < nb_active_devs; ++dev)
1197                 ret |= run_test_case_on_device(test_case_func, dev, op_params);
1198
1199         rte_free(op_params);
1200
1201         return ret;
1202 }
1203
1204 static void
1205 dequeue_event_callback(uint16_t dev_id,
1206                 enum rte_bbdev_event_type event, void *cb_arg,
1207                 void *ret_param)
1208 {
1209         int ret;
1210         uint16_t i;
1211         uint64_t total_time;
1212         uint16_t deq, burst_sz, num_ops;
1213         uint16_t queue_id = *(uint16_t *) ret_param;
1214         struct rte_bbdev_info info;
1215
1216         double tb_len_bits;
1217
1218         struct thread_params *tp = cb_arg;
1219
1220         /* Find matching thread params using queue_id */
1221         for (i = 0; i < MAX_QUEUES; ++i, ++tp)
1222                 if (tp->queue_id == queue_id)
1223                         break;
1224
1225         if (i == MAX_QUEUES) {
1226                 printf("%s: Queue_id from interrupt details was not found!\n",
1227                                 __func__);
1228                 return;
1229         }
1230
1231         if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
1232                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1233                 printf(
1234                         "Dequeue interrupt handler called for incorrect event!\n");
1235                 return;
1236         }
1237
1238         burst_sz = rte_atomic16_read(&tp->burst_sz);
1239         num_ops = tp->op_params->num_to_process;
1240
1241         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1242                 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1243                                 &tp->dec_ops[
1244                                         rte_atomic16_read(&tp->nb_dequeued)],
1245                                 burst_sz);
1246         else
1247                 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1248                                 &tp->enc_ops[
1249                                         rte_atomic16_read(&tp->nb_dequeued)],
1250                                 burst_sz);
1251
1252         if (deq < burst_sz) {
1253                 printf(
1254                         "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
1255                         burst_sz, deq);
1256                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1257                 return;
1258         }
1259
1260         if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
1261                 rte_atomic16_add(&tp->nb_dequeued, deq);
1262                 return;
1263         }
1264
1265         total_time = rte_rdtsc_precise() - tp->start_time;
1266
1267         rte_bbdev_info_get(dev_id, &info);
1268
1269         ret = TEST_SUCCESS;
1270
1271         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
1272                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1273                 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
1274                                 tp->op_params->vector_mask);
1275                 /* get the max of iter_count for all dequeued ops */
1276                 for (i = 0; i < num_ops; ++i)
1277                         tp->iter_count = RTE_MAX(
1278                                         tp->dec_ops[i]->turbo_dec.iter_count,
1279                                         tp->iter_count);
1280                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
1281         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
1282                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1283                 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
1284                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
1285         }
1286
1287         if (ret) {
1288                 printf("Buffers validation failed\n");
1289                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1290         }
1291
1292         switch (test_vector.op_type) {
1293         case RTE_BBDEV_OP_TURBO_DEC:
1294                 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
1295                 break;
1296         case RTE_BBDEV_OP_TURBO_ENC:
1297                 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
1298                 break;
1299         case RTE_BBDEV_OP_NONE:
1300                 tb_len_bits = 0.0;
1301                 break;
1302         default:
1303                 printf("Unknown op type: %d\n", test_vector.op_type);
1304                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1305                 return;
1306         }
1307
1308         tp->ops_per_sec += ((double)num_ops) /
1309                         ((double)total_time / (double)rte_get_tsc_hz());
1310         tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
1311                         ((double)total_time / (double)rte_get_tsc_hz());
1312
1313         rte_atomic16_add(&tp->nb_dequeued, deq);
1314 }
1315
1316 static int
1317 throughput_intr_lcore_dec(void *arg)
1318 {
1319         struct thread_params *tp = arg;
1320         unsigned int enqueued;
1321         const uint16_t queue_id = tp->queue_id;
1322         const uint16_t burst_sz = tp->op_params->burst_sz;
1323         const uint16_t num_to_process = tp->op_params->num_to_process;
1324         struct rte_bbdev_dec_op *ops[num_to_process];
1325         struct test_buffers *bufs = NULL;
1326         struct rte_bbdev_info info;
1327         int ret, i, j;
1328         uint16_t num_to_enq, enq;
1329
1330         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1331                         "BURST_SIZE should be <= %u", MAX_BURST);
1332
1333         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1334                         "Failed to enable interrupts for dev: %u, queue_id: %u",
1335                         tp->dev_id, queue_id);
1336
1337         rte_bbdev_info_get(tp->dev_id, &info);
1338
1339         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
1340                         "NUM_OPS cannot exceed %u for this device",
1341                         info.drv.queue_size_lim);
1342
1343         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1344
1345         rte_atomic16_clear(&tp->processing_status);
1346         rte_atomic16_clear(&tp->nb_dequeued);
1347
1348         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1349                 rte_pause();
1350
1351         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
1352                                 num_to_process);
1353         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
1354                         num_to_process);
1355         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1356                 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
1357                                 bufs->hard_outputs, bufs->soft_outputs,
1358                                 tp->op_params->ref_dec_op);
1359
1360         /* Set counter to validate the ordering */
1361         for (j = 0; j < num_to_process; ++j)
1362                 ops[j]->opaque_data = (void *)(uintptr_t)j;
1363
1364         for (j = 0; j < TEST_REPETITIONS; ++j) {
1365                 for (i = 0; i < num_to_process; ++i)
1366                         rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
1367
1368                 tp->start_time = rte_rdtsc_precise();
1369                 for (enqueued = 0; enqueued < num_to_process;) {
1370                         num_to_enq = burst_sz;
1371
1372                         if (unlikely(num_to_process - enqueued < num_to_enq))
1373                                 num_to_enq = num_to_process - enqueued;
1374
1375                         enq = 0;
1376                         do {
1377                                 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
1378                                         queue_id, &ops[enqueued],
1379                                         num_to_enq);
1380                         } while (unlikely(num_to_enq != enq));
1381                         enqueued += enq;
1382
1383                         /* Write to thread burst_sz current number of enqueued
1384                          * descriptors. It ensures that proper number of
1385                          * descriptors will be dequeued in callback
1386                          * function - needed for last batch in case where
1387                          * the number of operations is not a multiple of
1388                          * burst size.
1389                          */
1390                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
1391
1392                         /* Wait until processing of previous batch is
1393                          * completed.
1394                          */
1395                         while (rte_atomic16_read(&tp->nb_dequeued) !=
1396                                         (int16_t) enqueued)
1397                                 rte_pause();
1398                 }
1399                 if (j != TEST_REPETITIONS - 1)
1400                         rte_atomic16_clear(&tp->nb_dequeued);
1401         }
1402
1403         return TEST_SUCCESS;
1404 }
1405
1406 static int
1407 throughput_intr_lcore_enc(void *arg)
1408 {
1409         struct thread_params *tp = arg;
1410         unsigned int enqueued;
1411         const uint16_t queue_id = tp->queue_id;
1412         const uint16_t burst_sz = tp->op_params->burst_sz;
1413         const uint16_t num_to_process = tp->op_params->num_to_process;
1414         struct rte_bbdev_enc_op *ops[num_to_process];
1415         struct test_buffers *bufs = NULL;
1416         struct rte_bbdev_info info;
1417         int ret, i, j;
1418         uint16_t num_to_enq, enq;
1419
1420         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1421                         "BURST_SIZE should be <= %u", MAX_BURST);
1422
1423         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1424                         "Failed to enable interrupts for dev: %u, queue_id: %u",
1425                         tp->dev_id, queue_id);
1426
1427         rte_bbdev_info_get(tp->dev_id, &info);
1428
1429         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
1430                         "NUM_OPS cannot exceed %u for this device",
1431                         info.drv.queue_size_lim);
1432
1433         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1434
1435         rte_atomic16_clear(&tp->processing_status);
1436         rte_atomic16_clear(&tp->nb_dequeued);
1437
1438         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1439                 rte_pause();
1440
1441         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
1442                         num_to_process);
1443         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
1444                         num_to_process);
1445         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1446                 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
1447                                 bufs->hard_outputs, tp->op_params->ref_enc_op);
1448
1449         /* Set counter to validate the ordering */
1450         for (j = 0; j < num_to_process; ++j)
1451                 ops[j]->opaque_data = (void *)(uintptr_t)j;
1452
1453         for (j = 0; j < TEST_REPETITIONS; ++j) {
1454                 for (i = 0; i < num_to_process; ++i)
1455                         rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
1456
1457                 tp->start_time = rte_rdtsc_precise();
1458                 for (enqueued = 0; enqueued < num_to_process;) {
1459                         num_to_enq = burst_sz;
1460
1461                         if (unlikely(num_to_process - enqueued < num_to_enq))
1462                                 num_to_enq = num_to_process - enqueued;
1463
1464                         enq = 0;
1465                         do {
1466                                 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
1467                                                 queue_id, &ops[enqueued],
1468                                                 num_to_enq);
1469                         } while (unlikely(enq != num_to_enq));
1470                         enqueued += enq;
1471
1472                         /* Write to thread burst_sz current number of enqueued
1473                          * descriptors. It ensures that proper number of
1474                          * descriptors will be dequeued in callback
1475                          * function - needed for last batch in case where
1476                          * the number of operations is not a multiple of
1477                          * burst size.
1478                          */
1479                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
1480
1481                         /* Wait until processing of previous batch is
1482                          * completed.
1483                          */
1484                         while (rte_atomic16_read(&tp->nb_dequeued) !=
1485                                         (int16_t) enqueued)
1486                                 rte_pause();
1487                 }
1488                 if (j != TEST_REPETITIONS - 1)
1489                         rte_atomic16_clear(&tp->nb_dequeued);
1490         }
1491
1492         return TEST_SUCCESS;
1493 }
1494
1495 static int
1496 throughput_pmd_lcore_dec(void *arg)
1497 {
1498         struct thread_params *tp = arg;
1499         uint16_t enq, deq;
1500         uint64_t total_time = 0, start_time;
1501         const uint16_t queue_id = tp->queue_id;
1502         const uint16_t burst_sz = tp->op_params->burst_sz;
1503         const uint16_t num_ops = tp->op_params->num_to_process;
1504         struct rte_bbdev_dec_op *ops_enq[num_ops];
1505         struct rte_bbdev_dec_op *ops_deq[num_ops];
1506         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1507         struct test_buffers *bufs = NULL;
1508         int i, j, ret;
1509         struct rte_bbdev_info info;
1510         uint16_t num_to_enq;
1511
1512         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1513                         "BURST_SIZE should be <= %u", MAX_BURST);
1514
1515         rte_bbdev_info_get(tp->dev_id, &info);
1516
1517         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
1518                         "NUM_OPS cannot exceed %u for this device",
1519                         info.drv.queue_size_lim);
1520
1521         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1522
1523         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1524                 rte_pause();
1525
1526         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
1527         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
1528
1529         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1530                 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
1531                                 bufs->hard_outputs, bufs->soft_outputs, ref_op);
1532
1533         /* Set counter to validate the ordering */
1534         for (j = 0; j < num_ops; ++j)
1535                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1536
1537         for (i = 0; i < TEST_REPETITIONS; ++i) {
1538
1539                 for (j = 0; j < num_ops; ++j)
1540                         mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
1541
1542                 start_time = rte_rdtsc_precise();
1543
1544                 for (enq = 0, deq = 0; enq < num_ops;) {
1545                         num_to_enq = burst_sz;
1546
1547                         if (unlikely(num_ops - enq < num_to_enq))
1548                                 num_to_enq = num_ops - enq;
1549
1550                         enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
1551                                         queue_id, &ops_enq[enq], num_to_enq);
1552
1553                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
1554                                         queue_id, &ops_deq[deq], enq - deq);
1555                 }
1556
1557                 /* dequeue the remaining */
1558                 while (deq < enq) {
1559                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
1560                                         queue_id, &ops_deq[deq], enq - deq);
1561                 }
1562
1563                 total_time += rte_rdtsc_precise() - start_time;
1564         }
1565
1566         tp->iter_count = 0;
1567         /* get the max of iter_count for all dequeued ops */
1568         for (i = 0; i < num_ops; ++i) {
1569                 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
1570                                 tp->iter_count);
1571         }
1572
1573         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1574                 ret = validate_dec_op(ops_deq, num_ops, ref_op,
1575                                 tp->op_params->vector_mask);
1576                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1577         }
1578
1579         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
1580
1581         double tb_len_bits = calc_dec_TB_size(ref_op);
1582
1583         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
1584                         ((double)total_time / (double)rte_get_tsc_hz());
1585         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
1586                         1000000.0) / ((double)total_time /
1587                         (double)rte_get_tsc_hz());
1588
1589         return TEST_SUCCESS;
1590 }
1591
1592 static int
1593 throughput_pmd_lcore_enc(void *arg)
1594 {
1595         struct thread_params *tp = arg;
1596         uint16_t enq, deq;
1597         uint64_t total_time = 0, start_time;
1598         const uint16_t queue_id = tp->queue_id;
1599         const uint16_t burst_sz = tp->op_params->burst_sz;
1600         const uint16_t num_ops = tp->op_params->num_to_process;
1601         struct rte_bbdev_enc_op *ops_enq[num_ops];
1602         struct rte_bbdev_enc_op *ops_deq[num_ops];
1603         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1604         struct test_buffers *bufs = NULL;
1605         int i, j, ret;
1606         struct rte_bbdev_info info;
1607         uint16_t num_to_enq;
1608
1609         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1610                         "BURST_SIZE should be <= %u", MAX_BURST);
1611
1612         rte_bbdev_info_get(tp->dev_id, &info);
1613
1614         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
1615                         "NUM_OPS cannot exceed %u for this device",
1616                         info.drv.queue_size_lim);
1617
1618         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1619
1620         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1621                 rte_pause();
1622
1623         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
1624                         num_ops);
1625         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
1626                         num_ops);
1627         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1628                 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
1629                                 bufs->hard_outputs, ref_op);
1630
1631         /* Set counter to validate the ordering */
1632         for (j = 0; j < num_ops; ++j)
1633                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1634
1635         for (i = 0; i < TEST_REPETITIONS; ++i) {
1636
1637                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1638                         for (j = 0; j < num_ops; ++j)
1639                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
1640
1641                 start_time = rte_rdtsc_precise();
1642
1643                 for (enq = 0, deq = 0; enq < num_ops;) {
1644                         num_to_enq = burst_sz;
1645
1646                         if (unlikely(num_ops - enq < num_to_enq))
1647                                 num_to_enq = num_ops - enq;
1648
1649                         enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
1650                                         queue_id, &ops_enq[enq], num_to_enq);
1651
1652                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
1653                                         queue_id, &ops_deq[deq], enq - deq);
1654                 }
1655
1656                 /* dequeue the remaining */
1657                 while (deq < enq) {
1658                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
1659                                         queue_id, &ops_deq[deq], enq - deq);
1660                 }
1661
1662                 total_time += rte_rdtsc_precise() - start_time;
1663         }
1664
1665         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1666                 ret = validate_enc_op(ops_deq, num_ops, ref_op);
1667                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1668         }
1669
1670         double tb_len_bits = calc_enc_TB_size(ref_op);
1671
1672         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
1673                         ((double)total_time / (double)rte_get_tsc_hz());
1674         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
1675                         / 1000000.0) / ((double)total_time /
1676                         (double)rte_get_tsc_hz());
1677
1678         return TEST_SUCCESS;
1679 }
1680
1681 static void
1682 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
1683 {
1684         unsigned int iter = 0;
1685         double total_mops = 0, total_mbps = 0;
1686
1687         for (iter = 0; iter < used_cores; iter++) {
1688                 printf(
1689                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
1690                         t_params[iter].lcore_id, t_params[iter].ops_per_sec,
1691                         t_params[iter].mbps);
1692                 total_mops += t_params[iter].ops_per_sec;
1693                 total_mbps += t_params[iter].mbps;
1694         }
1695         printf(
1696                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
1697                 used_cores, total_mops, total_mbps);
1698 }
1699
1700 static void
1701 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
1702 {
1703         unsigned int iter = 0;
1704         double total_mops = 0, total_mbps = 0;
1705         uint8_t iter_count = 0;
1706
1707         for (iter = 0; iter < used_cores; iter++) {
1708                 printf(
1709                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
1710                         t_params[iter].lcore_id, t_params[iter].ops_per_sec,
1711                         t_params[iter].mbps, t_params[iter].iter_count);
1712                 total_mops += t_params[iter].ops_per_sec;
1713                 total_mbps += t_params[iter].mbps;
1714                 iter_count = RTE_MAX(iter_count, t_params[iter].iter_count);
1715         }
1716         printf(
1717                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
1718                 used_cores, total_mops, total_mbps, iter_count);
1719 }
1720
1721 /*
1722  * Test function that determines how long an enqueue + dequeue of a burst
1723  * takes on available lcores.
1724  */
1725 static int
1726 throughput_test(struct active_device *ad,
1727                 struct test_op_params *op_params)
1728 {
1729         int ret;
1730         unsigned int lcore_id, used_cores = 0;
1731         struct thread_params *t_params, *tp;
1732         struct rte_bbdev_info info;
1733         lcore_function_t *throughput_function;
1734         uint16_t num_lcores;
1735         const char *op_type_str;
1736
1737         rte_bbdev_info_get(ad->dev_id, &info);
1738
1739         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
1740         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
1741                         test_vector.op_type);
1742
1743         printf(
1744                 "Throughput test: dev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, int mode: %s, GHz: %lg\n",
1745                         info.dev_name, ad->nb_queues, op_params->burst_sz,
1746                         op_params->num_to_process, op_params->num_lcores,
1747                         op_type_str,
1748                         intr_enabled ? "Interrupt mode" : "PMD mode",
1749                         (double)rte_get_tsc_hz() / 1000000000.0);
1750
1751         /* Set number of lcores */
1752         num_lcores = (ad->nb_queues < (op_params->num_lcores))
1753                         ? ad->nb_queues
1754                         : op_params->num_lcores;
1755
1756         /* Allocate memory for thread parameters structure */
1757         t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
1758                         RTE_CACHE_LINE_SIZE);
1759         TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
1760                         RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
1761                                 RTE_CACHE_LINE_SIZE));
1762
1763         if (intr_enabled) {
1764                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1765                         throughput_function = throughput_intr_lcore_dec;
1766                 else
1767                         throughput_function = throughput_intr_lcore_enc;
1768
1769                 /* Dequeue interrupt callback registration */
1770                 ret = rte_bbdev_callback_register(ad->dev_id,
1771                                 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
1772                                 t_params);
1773                 if (ret < 0) {
1774                         rte_free(t_params);
1775                         return ret;
1776                 }
1777         } else {
1778                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1779                         throughput_function = throughput_pmd_lcore_dec;
1780                 else
1781                         throughput_function = throughput_pmd_lcore_enc;
1782         }
1783
1784         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
1785
1786         /* Master core is set at first entry */
1787         t_params[0].dev_id = ad->dev_id;
1788         t_params[0].lcore_id = rte_lcore_id();
1789         t_params[0].op_params = op_params;
1790         t_params[0].queue_id = ad->queue_ids[used_cores++];
1791         t_params[0].iter_count = 0;
1792
1793         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1794                 if (used_cores >= num_lcores)
1795                         break;
1796
1797                 t_params[used_cores].dev_id = ad->dev_id;
1798                 t_params[used_cores].lcore_id = lcore_id;
1799                 t_params[used_cores].op_params = op_params;
1800                 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
1801                 t_params[used_cores].iter_count = 0;
1802
1803                 rte_eal_remote_launch(throughput_function,
1804                                 &t_params[used_cores++], lcore_id);
1805         }
1806
1807         rte_atomic16_set(&op_params->sync, SYNC_START);
1808         ret = throughput_function(&t_params[0]);
1809
1810         /* Master core is always used */
1811         for (used_cores = 1; used_cores < num_lcores; used_cores++)
1812                 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
1813
1814         /* Return if test failed */
1815         if (ret) {
1816                 rte_free(t_params);
1817                 return ret;
1818         }
1819
1820         /* Print throughput if interrupts are disabled and test passed */
1821         if (!intr_enabled) {
1822                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1823                         print_dec_throughput(t_params, num_lcores);
1824                 else
1825                         print_enc_throughput(t_params, num_lcores);
1826                 rte_free(t_params);
1827                 return ret;
1828         }
1829
1830         /* In interrupt TC we need to wait for the interrupt callback to deqeue
1831          * all pending operations. Skip waiting for queues which reported an
1832          * error using processing_status variable.
1833          * Wait for master lcore operations.
1834          */
1835         tp = &t_params[0];
1836         while ((rte_atomic16_read(&tp->nb_dequeued) <
1837                         op_params->num_to_process) &&
1838                         (rte_atomic16_read(&tp->processing_status) !=
1839                         TEST_FAILED))
1840                 rte_pause();
1841
1842         tp->ops_per_sec /= TEST_REPETITIONS;
1843         tp->mbps /= TEST_REPETITIONS;
1844         ret |= rte_atomic16_read(&tp->processing_status);
1845
1846         /* Wait for slave lcores operations */
1847         for (used_cores = 1; used_cores < num_lcores; used_cores++) {
1848                 tp = &t_params[used_cores];
1849
1850                 while ((rte_atomic16_read(&tp->nb_dequeued) <
1851                                 op_params->num_to_process) &&
1852                                 (rte_atomic16_read(&tp->processing_status) !=
1853                                 TEST_FAILED))
1854                         rte_pause();
1855
1856                 tp->ops_per_sec /= TEST_REPETITIONS;
1857                 tp->mbps /= TEST_REPETITIONS;
1858                 ret |= rte_atomic16_read(&tp->processing_status);
1859         }
1860
1861         /* Print throughput if test passed */
1862         if (!ret) {
1863                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1864                         print_dec_throughput(t_params, num_lcores);
1865                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1866                         print_enc_throughput(t_params, num_lcores);
1867         }
1868
1869         rte_free(t_params);
1870         return ret;
1871 }
1872
1873 static int
1874 latency_test_dec(struct rte_mempool *mempool,
1875                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
1876                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
1877                 const uint16_t num_to_process, uint16_t burst_sz,
1878                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
1879 {
1880         int ret = TEST_SUCCESS;
1881         uint16_t i, j, dequeued;
1882         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1883         uint64_t start_time = 0, last_time = 0;
1884
1885         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1886                 uint16_t enq = 0, deq = 0;
1887                 bool first_time = true;
1888                 last_time = 0;
1889
1890                 if (unlikely(num_to_process - dequeued < burst_sz))
1891                         burst_sz = num_to_process - dequeued;
1892
1893                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1894                 TEST_ASSERT_SUCCESS(ret,
1895                                 "rte_bbdev_dec_op_alloc_bulk() failed");
1896                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1897                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1898                                         bufs->inputs,
1899                                         bufs->hard_outputs,
1900                                         bufs->soft_outputs,
1901                                         ref_op);
1902
1903                 /* Set counter to validate the ordering */
1904                 for (j = 0; j < burst_sz; ++j)
1905                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1906
1907                 start_time = rte_rdtsc_precise();
1908
1909                 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
1910                                 burst_sz);
1911                 TEST_ASSERT(enq == burst_sz,
1912                                 "Error enqueueing burst, expected %u, got %u",
1913                                 burst_sz, enq);
1914
1915                 /* Dequeue */
1916                 do {
1917                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1918                                         &ops_deq[deq], burst_sz - deq);
1919                         if (likely(first_time && (deq > 0))) {
1920                                 last_time = rte_rdtsc_precise() - start_time;
1921                                 first_time = false;
1922                         }
1923                 } while (unlikely(burst_sz != deq));
1924
1925                 *max_time = RTE_MAX(*max_time, last_time);
1926                 *min_time = RTE_MIN(*min_time, last_time);
1927                 *total_time += last_time;
1928
1929                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1930                         ret = validate_dec_op(ops_deq, burst_sz, ref_op,
1931                                         vector_mask);
1932                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1933                 }
1934
1935                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1936                 dequeued += deq;
1937         }
1938
1939         return i;
1940 }
1941
1942 static int
1943 latency_test_enc(struct rte_mempool *mempool,
1944                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
1945                 uint16_t dev_id, uint16_t queue_id,
1946                 const uint16_t num_to_process, uint16_t burst_sz,
1947                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
1948 {
1949         int ret = TEST_SUCCESS;
1950         uint16_t i, j, dequeued;
1951         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1952         uint64_t start_time = 0, last_time = 0;
1953
1954         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1955                 uint16_t enq = 0, deq = 0;
1956                 bool first_time = true;
1957                 last_time = 0;
1958
1959                 if (unlikely(num_to_process - dequeued < burst_sz))
1960                         burst_sz = num_to_process - dequeued;
1961
1962                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
1963                 TEST_ASSERT_SUCCESS(ret,
1964                                 "rte_bbdev_enc_op_alloc_bulk() failed");
1965                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1966                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
1967                                         bufs->inputs,
1968                                         bufs->hard_outputs,
1969                                         ref_op);
1970
1971                 /* Set counter to validate the ordering */
1972                 for (j = 0; j < burst_sz; ++j)
1973                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1974
1975                 start_time = rte_rdtsc_precise();
1976
1977                 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
1978                                 burst_sz);
1979                 TEST_ASSERT(enq == burst_sz,
1980                                 "Error enqueueing burst, expected %u, got %u",
1981                                 burst_sz, enq);
1982
1983                 /* Dequeue */
1984                 do {
1985                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1986                                         &ops_deq[deq], burst_sz - deq);
1987                         if (likely(first_time && (deq > 0))) {
1988                                 last_time += rte_rdtsc_precise() - start_time;
1989                                 first_time = false;
1990                         }
1991                 } while (unlikely(burst_sz != deq));
1992
1993                 *max_time = RTE_MAX(*max_time, last_time);
1994                 *min_time = RTE_MIN(*min_time, last_time);
1995                 *total_time += last_time;
1996
1997                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1998                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
1999                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2000                 }
2001
2002                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
2003                 dequeued += deq;
2004         }
2005
2006         return i;
2007 }
2008
2009 static int
2010 latency_test(struct active_device *ad,
2011                 struct test_op_params *op_params)
2012 {
2013         int iter;
2014         uint16_t burst_sz = op_params->burst_sz;
2015         const uint16_t num_to_process = op_params->num_to_process;
2016         const enum rte_bbdev_op_type op_type = test_vector.op_type;
2017         const uint16_t queue_id = ad->queue_ids[0];
2018         struct test_buffers *bufs = NULL;
2019         struct rte_bbdev_info info;
2020         uint64_t total_time, min_time, max_time;
2021         const char *op_type_str;
2022
2023         total_time = max_time = 0;
2024         min_time = UINT64_MAX;
2025
2026         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2027                         "BURST_SIZE should be <= %u", MAX_BURST);
2028
2029         rte_bbdev_info_get(ad->dev_id, &info);
2030         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2031
2032         op_type_str = rte_bbdev_op_type_str(op_type);
2033         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2034
2035         printf(
2036                 "\nValidation/Latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
2037                         info.dev_name, burst_sz, num_to_process, op_type_str);
2038
2039         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2040                 iter = latency_test_dec(op_params->mp, bufs,
2041                                 op_params->ref_dec_op, op_params->vector_mask,
2042                                 ad->dev_id, queue_id, num_to_process,
2043                                 burst_sz, &total_time, &min_time, &max_time);
2044         else
2045                 iter = latency_test_enc(op_params->mp, bufs,
2046                                 op_params->ref_enc_op, ad->dev_id, queue_id,
2047                                 num_to_process, burst_sz, &total_time,
2048                                 &min_time, &max_time);
2049
2050         if (iter <= 0)
2051                 return TEST_FAILED;
2052
2053         printf("Operation latency:\n"
2054                         "\tavg latency: %lg cycles, %lg us\n"
2055                         "\tmin latency: %lg cycles, %lg us\n"
2056                         "\tmax latency: %lg cycles, %lg us\n",
2057                         (double)total_time / (double)iter,
2058                         (double)(total_time * 1000000) / (double)iter /
2059                         (double)rte_get_tsc_hz(), (double)min_time,
2060                         (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
2061                         (double)max_time, (double)(max_time * 1000000) /
2062                         (double)rte_get_tsc_hz());
2063
2064         return TEST_SUCCESS;
2065 }
2066
2067 #ifdef RTE_BBDEV_OFFLOAD_COST
2068 static int
2069 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
2070                 struct rte_bbdev_stats *stats)
2071 {
2072         struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
2073         struct rte_bbdev_stats *q_stats;
2074
2075         if (queue_id >= dev->data->num_queues)
2076                 return -1;
2077
2078         q_stats = &dev->data->queues[queue_id].queue_stats;
2079
2080         stats->enqueued_count = q_stats->enqueued_count;
2081         stats->dequeued_count = q_stats->dequeued_count;
2082         stats->enqueue_err_count = q_stats->enqueue_err_count;
2083         stats->dequeue_err_count = q_stats->dequeue_err_count;
2084         stats->acc_offload_cycles = q_stats->acc_offload_cycles;
2085
2086         return 0;
2087 }
2088
2089 static int
2090 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
2091                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
2092                 uint16_t queue_id, const uint16_t num_to_process,
2093                 uint16_t burst_sz, struct test_time_stats *time_st)
2094 {
2095         int i, dequeued, ret;
2096         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2097         uint64_t enq_start_time, deq_start_time;
2098         uint64_t enq_sw_last_time, deq_last_time;
2099         struct rte_bbdev_stats stats;
2100
2101         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2102                 uint16_t enq = 0, deq = 0;
2103
2104                 if (unlikely(num_to_process - dequeued < burst_sz))
2105                         burst_sz = num_to_process - dequeued;
2106
2107                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
2108                 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2109                                 burst_sz);
2110
2111                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2112                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
2113                                         bufs->inputs,
2114                                         bufs->hard_outputs,
2115                                         bufs->soft_outputs,
2116                                         ref_op);
2117
2118                 /* Start time meas for enqueue function offload latency */
2119                 enq_start_time = rte_rdtsc_precise();
2120                 do {
2121                         enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
2122                                         &ops_enq[enq], burst_sz - enq);
2123                 } while (unlikely(burst_sz != enq));
2124
2125                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
2126                 TEST_ASSERT_SUCCESS(ret,
2127                                 "Failed to get stats for queue (%u) of device (%u)",
2128                                 queue_id, dev_id);
2129
2130                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
2131                                 stats.acc_offload_cycles;
2132                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
2133                                 enq_sw_last_time);
2134                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
2135                                 enq_sw_last_time);
2136                 time_st->enq_sw_total_time += enq_sw_last_time;
2137
2138                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
2139                                 stats.acc_offload_cycles);
2140                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
2141                                 stats.acc_offload_cycles);
2142                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
2143
2144                 /* ensure enqueue has been completed */
2145                 rte_delay_us(200);
2146
2147                 /* Start time meas for dequeue function offload latency */
2148                 deq_start_time = rte_rdtsc_precise();
2149                 /* Dequeue one operation */
2150                 do {
2151                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2152                                         &ops_deq[deq], 1);
2153                 } while (unlikely(deq != 1));
2154
2155                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
2156                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
2157                                 deq_last_time);
2158                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
2159                                 deq_last_time);
2160                 time_st->deq_total_time += deq_last_time;
2161
2162                 /* Dequeue remaining operations if needed*/
2163                 while (burst_sz != deq)
2164                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2165                                         &ops_deq[deq], burst_sz - deq);
2166
2167                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
2168                 dequeued += deq;
2169         }
2170
2171         return i;
2172 }
2173
2174 static int
2175 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
2176                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
2177                 uint16_t queue_id, const uint16_t num_to_process,
2178                 uint16_t burst_sz, struct test_time_stats *time_st)
2179 {
2180         int i, dequeued, ret;
2181         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2182         uint64_t enq_start_time, deq_start_time;
2183         uint64_t enq_sw_last_time, deq_last_time;
2184         struct rte_bbdev_stats stats;
2185
2186         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2187                 uint16_t enq = 0, deq = 0;
2188
2189                 if (unlikely(num_to_process - dequeued < burst_sz))
2190                         burst_sz = num_to_process - dequeued;
2191
2192                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
2193                 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2194                                 burst_sz);
2195
2196                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2197                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
2198                                         bufs->inputs,
2199                                         bufs->hard_outputs,
2200                                         ref_op);
2201
2202                 /* Start time meas for enqueue function offload latency */
2203                 enq_start_time = rte_rdtsc_precise();
2204                 do {
2205                         enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
2206                                         &ops_enq[enq], burst_sz - enq);
2207                 } while (unlikely(burst_sz != enq));
2208
2209                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
2210                 TEST_ASSERT_SUCCESS(ret,
2211                                 "Failed to get stats for queue (%u) of device (%u)",
2212                                 queue_id, dev_id);
2213
2214                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
2215                                 stats.acc_offload_cycles;
2216                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
2217                                 enq_sw_last_time);
2218                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
2219                                 enq_sw_last_time);
2220                 time_st->enq_sw_total_time += enq_sw_last_time;
2221
2222                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
2223                                 stats.acc_offload_cycles);
2224                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
2225                                 stats.acc_offload_cycles);
2226                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
2227
2228                 /* ensure enqueue has been completed */
2229                 rte_delay_us(200);
2230
2231                 /* Start time meas for dequeue function offload latency */
2232                 deq_start_time = rte_rdtsc_precise();
2233                 /* Dequeue one operation */
2234                 do {
2235                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2236                                         &ops_deq[deq], 1);
2237                 } while (unlikely(deq != 1));
2238
2239                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
2240                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
2241                                 deq_last_time);
2242                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
2243                                 deq_last_time);
2244                 time_st->deq_total_time += deq_last_time;
2245
2246                 while (burst_sz != deq)
2247                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2248                                         &ops_deq[deq], burst_sz - deq);
2249
2250                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
2251                 dequeued += deq;
2252         }
2253
2254         return i;
2255 }
2256 #endif
2257
2258 static int
2259 offload_cost_test(struct active_device *ad,
2260                 struct test_op_params *op_params)
2261 {
2262 #ifndef RTE_BBDEV_OFFLOAD_COST
2263         RTE_SET_USED(ad);
2264         RTE_SET_USED(op_params);
2265         printf("Offload latency test is disabled.\n");
2266         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
2267         return TEST_SKIPPED;
2268 #else
2269         int iter;
2270         uint16_t burst_sz = op_params->burst_sz;
2271         const uint16_t num_to_process = op_params->num_to_process;
2272         const enum rte_bbdev_op_type op_type = test_vector.op_type;
2273         const uint16_t queue_id = ad->queue_ids[0];
2274         struct test_buffers *bufs = NULL;
2275         struct rte_bbdev_info info;
2276         const char *op_type_str;
2277         struct test_time_stats time_st;
2278
2279         memset(&time_st, 0, sizeof(struct test_time_stats));
2280         time_st.enq_sw_min_time = UINT64_MAX;
2281         time_st.enq_acc_min_time = UINT64_MAX;
2282         time_st.deq_min_time = UINT64_MAX;
2283
2284         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2285                         "BURST_SIZE should be <= %u", MAX_BURST);
2286
2287         rte_bbdev_info_get(ad->dev_id, &info);
2288         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2289
2290         op_type_str = rte_bbdev_op_type_str(op_type);
2291         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2292
2293         printf(
2294                 "\nOffload latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
2295                         info.dev_name, burst_sz, num_to_process, op_type_str);
2296
2297         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2298                 iter = offload_latency_test_dec(op_params->mp, bufs,
2299                                 op_params->ref_dec_op, ad->dev_id, queue_id,
2300                                 num_to_process, burst_sz, &time_st);
2301         else
2302                 iter = offload_latency_test_enc(op_params->mp, bufs,
2303                                 op_params->ref_enc_op, ad->dev_id, queue_id,
2304                                 num_to_process, burst_sz, &time_st);
2305
2306         if (iter <= 0)
2307                 return TEST_FAILED;
2308
2309         printf("Enqueue offload cost latency:\n"
2310                         "\tDriver offload avg %lg cycles, %lg us\n"
2311                         "\tDriver offload min %lg cycles, %lg us\n"
2312                         "\tDriver offload max %lg cycles, %lg us\n"
2313                         "\tAccelerator offload avg %lg cycles, %lg us\n"
2314                         "\tAccelerator offload min %lg cycles, %lg us\n"
2315                         "\tAccelerator offload max %lg cycles, %lg us\n",
2316                         (double)time_st.enq_sw_total_time / (double)iter,
2317                         (double)(time_st.enq_sw_total_time * 1000000) /
2318                         (double)iter / (double)rte_get_tsc_hz(),
2319                         (double)time_st.enq_sw_min_time,
2320                         (double)(time_st.enq_sw_min_time * 1000000) /
2321                         rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
2322                         (double)(time_st.enq_sw_max_time * 1000000) /
2323                         rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
2324                         (double)iter,
2325                         (double)(time_st.enq_acc_total_time * 1000000) /
2326                         (double)iter / (double)rte_get_tsc_hz(),
2327                         (double)time_st.enq_acc_min_time,
2328                         (double)(time_st.enq_acc_min_time * 1000000) /
2329                         rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
2330                         (double)(time_st.enq_acc_max_time * 1000000) /
2331                         rte_get_tsc_hz());
2332
2333         printf("Dequeue offload cost latency - one op:\n"
2334                         "\tavg %lg cycles, %lg us\n"
2335                         "\tmin %lg cycles, %lg us\n"
2336                         "\tmax %lg cycles, %lg us\n",
2337                         (double)time_st.deq_total_time / (double)iter,
2338                         (double)(time_st.deq_total_time * 1000000) /
2339                         (double)iter / (double)rte_get_tsc_hz(),
2340                         (double)time_st.deq_min_time,
2341                         (double)(time_st.deq_min_time * 1000000) /
2342                         rte_get_tsc_hz(), (double)time_st.deq_max_time,
2343                         (double)(time_st.deq_max_time * 1000000) /
2344                         rte_get_tsc_hz());
2345
2346         return TEST_SUCCESS;
2347 #endif
2348 }
2349
2350 #ifdef RTE_BBDEV_OFFLOAD_COST
2351 static int
2352 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
2353                 const uint16_t num_to_process, uint16_t burst_sz,
2354                 uint64_t *deq_total_time, uint64_t *deq_min_time,
2355                 uint64_t *deq_max_time)
2356 {
2357         int i, deq_total;
2358         struct rte_bbdev_dec_op *ops[MAX_BURST];
2359         uint64_t deq_start_time, deq_last_time;
2360
2361         /* Test deq offload latency from an empty queue */
2362
2363         for (i = 0, deq_total = 0; deq_total < num_to_process;
2364                         ++i, deq_total += burst_sz) {
2365                 deq_start_time = rte_rdtsc_precise();
2366
2367                 if (unlikely(num_to_process - deq_total < burst_sz))
2368                         burst_sz = num_to_process - deq_total;
2369                 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz);
2370
2371                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
2372                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
2373                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
2374                 *deq_total_time += deq_last_time;
2375         }
2376
2377         return i;
2378 }
2379
2380 static int
2381 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
2382                 const uint16_t num_to_process, uint16_t burst_sz,
2383                 uint64_t *deq_total_time, uint64_t *deq_min_time,
2384                 uint64_t *deq_max_time)
2385 {
2386         int i, deq_total;
2387         struct rte_bbdev_enc_op *ops[MAX_BURST];
2388         uint64_t deq_start_time, deq_last_time;
2389
2390         /* Test deq offload latency from an empty queue */
2391         for (i = 0, deq_total = 0; deq_total < num_to_process;
2392                         ++i, deq_total += burst_sz) {
2393                 deq_start_time = rte_rdtsc_precise();
2394
2395                 if (unlikely(num_to_process - deq_total < burst_sz))
2396                         burst_sz = num_to_process - deq_total;
2397                 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz);
2398
2399                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
2400                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
2401                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
2402                 *deq_total_time += deq_last_time;
2403         }
2404
2405         return i;
2406 }
2407 #endif
2408
2409 static int
2410 offload_latency_empty_q_test(struct active_device *ad,
2411                 struct test_op_params *op_params)
2412 {
2413 #ifndef RTE_BBDEV_OFFLOAD_COST
2414         RTE_SET_USED(ad);
2415         RTE_SET_USED(op_params);
2416         printf("Offload latency empty dequeue test is disabled.\n");
2417         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
2418         return TEST_SKIPPED;
2419 #else
2420         int iter;
2421         uint64_t deq_total_time, deq_min_time, deq_max_time;
2422         uint16_t burst_sz = op_params->burst_sz;
2423         const uint16_t num_to_process = op_params->num_to_process;
2424         const enum rte_bbdev_op_type op_type = test_vector.op_type;
2425         const uint16_t queue_id = ad->queue_ids[0];
2426         struct rte_bbdev_info info;
2427         const char *op_type_str;
2428
2429         deq_total_time = deq_max_time = 0;
2430         deq_min_time = UINT64_MAX;
2431
2432         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2433                         "BURST_SIZE should be <= %u", MAX_BURST);
2434
2435         rte_bbdev_info_get(ad->dev_id, &info);
2436
2437         op_type_str = rte_bbdev_op_type_str(op_type);
2438         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2439
2440         printf(
2441                 "\nOffload latency empty dequeue test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
2442                         info.dev_name, burst_sz, num_to_process, op_type_str);
2443
2444         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2445                 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
2446                                 num_to_process, burst_sz, &deq_total_time,
2447                                 &deq_min_time, &deq_max_time);
2448         else
2449                 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
2450                                 num_to_process, burst_sz, &deq_total_time,
2451                                 &deq_min_time, &deq_max_time);
2452
2453         if (iter <= 0)
2454                 return TEST_FAILED;
2455
2456         printf("Empty dequeue offload\n"
2457                         "\tavg. latency: %lg cycles, %lg us\n"
2458                         "\tmin. latency: %lg cycles, %lg us\n"
2459                         "\tmax. latency: %lg cycles, %lg us\n",
2460                         (double)deq_total_time / (double)iter,
2461                         (double)(deq_total_time * 1000000) / (double)iter /
2462                         (double)rte_get_tsc_hz(), (double)deq_min_time,
2463                         (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
2464                         (double)deq_max_time, (double)(deq_max_time * 1000000) /
2465                         rte_get_tsc_hz());
2466
2467         return TEST_SUCCESS;
2468 #endif
2469 }
2470
2471 static int
2472 throughput_tc(void)
2473 {
2474         return run_test_case(throughput_test);
2475 }
2476
2477 static int
2478 offload_cost_tc(void)
2479 {
2480         return run_test_case(offload_cost_test);
2481 }
2482
2483 static int
2484 offload_latency_empty_q_tc(void)
2485 {
2486         return run_test_case(offload_latency_empty_q_test);
2487 }
2488
2489 static int
2490 latency_tc(void)
2491 {
2492         return run_test_case(latency_test);
2493 }
2494
2495 static int
2496 interrupt_tc(void)
2497 {
2498         return run_test_case(throughput_test);
2499 }
2500
2501 static struct unit_test_suite bbdev_throughput_testsuite = {
2502         .suite_name = "BBdev Throughput Tests",
2503         .setup = testsuite_setup,
2504         .teardown = testsuite_teardown,
2505         .unit_test_cases = {
2506                 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
2507                 TEST_CASES_END() /**< NULL terminate unit test array */
2508         }
2509 };
2510
2511 static struct unit_test_suite bbdev_validation_testsuite = {
2512         .suite_name = "BBdev Validation Tests",
2513         .setup = testsuite_setup,
2514         .teardown = testsuite_teardown,
2515         .unit_test_cases = {
2516                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
2517                 TEST_CASES_END() /**< NULL terminate unit test array */
2518         }
2519 };
2520
2521 static struct unit_test_suite bbdev_latency_testsuite = {
2522         .suite_name = "BBdev Latency Tests",
2523         .setup = testsuite_setup,
2524         .teardown = testsuite_teardown,
2525         .unit_test_cases = {
2526                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
2527                 TEST_CASES_END() /**< NULL terminate unit test array */
2528         }
2529 };
2530
2531 static struct unit_test_suite bbdev_offload_cost_testsuite = {
2532         .suite_name = "BBdev Offload Cost Tests",
2533         .setup = testsuite_setup,
2534         .teardown = testsuite_teardown,
2535         .unit_test_cases = {
2536                 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
2537                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
2538                 TEST_CASES_END() /**< NULL terminate unit test array */
2539         }
2540 };
2541
2542 static struct unit_test_suite bbdev_interrupt_testsuite = {
2543         .suite_name = "BBdev Interrupt Tests",
2544         .setup = interrupt_testsuite_setup,
2545         .teardown = testsuite_teardown,
2546         .unit_test_cases = {
2547                 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
2548                 TEST_CASES_END() /**< NULL terminate unit test array */
2549         }
2550 };
2551
2552 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
2553 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
2554 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
2555 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
2556 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);