app/test-bbdev/test_bbdev_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7 #include <math.h>
   8
   9 #include <rte_eal.h>
  10 #include <rte_common.h>
  11 #include <rte_dev.h>
  12 #include <rte_launch.h>
  13 #include <rte_bbdev.h>
  14 #include <rte_cycles.h>
  15 #include <rte_lcore.h>
  16 #include <rte_malloc.h>
  17 #include <rte_random.h>
  18 #include <rte_hexdump.h>
  19 #include <rte_interrupts.h>
  20
  21 #include "main.h"
  22 #include "test_bbdev_vector.h"
  23
  24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
  25
  26 #define MAX_QUEUES RTE_MAX_LCORE
  27 #define TEST_REPETITIONS 1000
  28
  29 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
  30 #include <fpga_lte_fec.h>
  31 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
  32 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
  33 #define VF_UL_4G_QUEUE_VALUE 4
  34 #define VF_DL_4G_QUEUE_VALUE 4
  35 #define UL_4G_BANDWIDTH 3
  36 #define DL_4G_BANDWIDTH 3
  37 #define UL_4G_LOAD_BALANCE 128
  38 #define DL_4G_LOAD_BALANCE 128
  39 #define FLR_4G_TIMEOUT 610
  40 #endif
  41
  42 #define OPS_CACHE_SIZE 256U
  43 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
  44
  45 #define SYNC_WAIT 0
  46 #define SYNC_START 1
  47
  48 #define INVALID_QUEUE_ID -1
  49
  50 static struct test_bbdev_vector test_vector;
  51
  52 /* Switch between PMD and Interrupt for throughput TC */
  53 static bool intr_enabled;
  54
  55 /* Represents tested active devices */
  56 static struct active_device {
  57         const char *driver_name;
  58         uint8_t dev_id;
  59         uint16_t supported_ops;
  60         uint16_t queue_ids[MAX_QUEUES];
  61         uint16_t nb_queues;
  62         struct rte_mempool *ops_mempool;
  63         struct rte_mempool *in_mbuf_pool;
  64         struct rte_mempool *hard_out_mbuf_pool;
  65         struct rte_mempool *soft_out_mbuf_pool;
  66         struct rte_mempool *harq_in_mbuf_pool;
  67         struct rte_mempool *harq_out_mbuf_pool;
  68 } active_devs[RTE_BBDEV_MAX_DEVS];
  69
  70 static uint8_t nb_active_devs;
  71
  72 /* Data buffers used by BBDEV ops */
  73 struct test_buffers {
  74         struct rte_bbdev_op_data *inputs;
  75         struct rte_bbdev_op_data *hard_outputs;
  76         struct rte_bbdev_op_data *soft_outputs;
  77         struct rte_bbdev_op_data *harq_inputs;
  78         struct rte_bbdev_op_data *harq_outputs;
  79 };
  80
  81 /* Operation parameters specific for given test case */
  82 struct test_op_params {
  83         struct rte_mempool *mp;
  84         struct rte_bbdev_dec_op *ref_dec_op;
  85         struct rte_bbdev_enc_op *ref_enc_op;
  86         uint16_t burst_sz;
  87         uint16_t num_to_process;
  88         uint16_t num_lcores;
  89         int vector_mask;
  90         rte_atomic16_t sync;
  91         struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
  92 };
  93
  94 /* Contains per lcore params */
  95 struct thread_params {
  96         uint8_t dev_id;
  97         uint16_t queue_id;
  98         uint32_t lcore_id;
  99         uint64_t start_time;
 100         double ops_per_sec;
 101         double mbps;
 102         uint8_t iter_count;
 103         rte_atomic16_t nb_dequeued;
 104         rte_atomic16_t processing_status;
 105         rte_atomic16_t burst_sz;
 106         struct test_op_params *op_params;
 107         struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
 108         struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
 109 };
 110
 111 #ifdef RTE_BBDEV_OFFLOAD_COST
 112 /* Stores time statistics */
 113 struct test_time_stats {
 114         /* Stores software enqueue total working time */
 115         uint64_t enq_sw_total_time;
 116         /* Stores minimum value of software enqueue working time */
 117         uint64_t enq_sw_min_time;
 118         /* Stores maximum value of software enqueue working time */
 119         uint64_t enq_sw_max_time;
 120         /* Stores turbo enqueue total working time */
 121         uint64_t enq_acc_total_time;
 122         /* Stores minimum value of accelerator enqueue working time */
 123         uint64_t enq_acc_min_time;
 124         /* Stores maximum value of accelerator enqueue working time */
 125         uint64_t enq_acc_max_time;
 126         /* Stores dequeue total working time */
 127         uint64_t deq_total_time;
 128         /* Stores minimum value of dequeue working time */
 129         uint64_t deq_min_time;
 130         /* Stores maximum value of dequeue working time */
 131         uint64_t deq_max_time;
 132 };
 133 #endif
 134
 135 typedef int (test_case_function)(struct active_device *ad,
 136                 struct test_op_params *op_params);
 137
 138 static inline void
 139 mbuf_reset(struct rte_mbuf *m)
 140 {
 141         m->pkt_len = 0;
 142
 143         do {
 144                 m->data_len = 0;
 145                 m = m->next;
 146         } while (m != NULL);
 147 }
 148
 149 /* Read flag value 0/1 from bitmap */
 150 static inline bool
 151 check_bit(uint32_t bitmap, uint32_t bitmask)
 152 {
 153         return bitmap & bitmask;
 154 }
 155
 156 static inline void
 157 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 158 {
 159         ad->supported_ops |= (1 << op_type);
 160 }
 161
 162 static inline bool
 163 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 164 {
 165         return ad->supported_ops & (1 << op_type);
 166 }
 167
 168 static inline bool
 169 flags_match(uint32_t flags_req, uint32_t flags_present)
 170 {
 171         return (flags_req & flags_present) == flags_req;
 172 }
 173
 174 static void
 175 clear_soft_out_cap(uint32_t *op_flags)
 176 {
 177         *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
 178         *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
 179         *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
 180 }
 181
 182 static int
 183 check_dev_cap(const struct rte_bbdev_info *dev_info)
 184 {
 185         unsigned int i;
 186         unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
 187                 nb_harq_inputs, nb_harq_outputs;
 188         const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
 189
 190         nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
 191         nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
 192         nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
 193         nb_harq_inputs  = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
 194         nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
 195
 196         for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
 197                 if (op_cap->type != test_vector.op_type)
 198                         continue;
 199
 200                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
 201                         const struct rte_bbdev_op_cap_turbo_dec *cap =
 202                                         &op_cap->cap.turbo_dec;
 203                         /* Ignore lack of soft output capability, just skip
 204                          * checking if soft output is valid.
 205                          */
 206                         if ((test_vector.turbo_dec.op_flags &
 207                                         RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
 208                                         !(cap->capability_flags &
 209                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 210                                 printf(
 211                                         "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
 212                                         dev_info->dev_name);
 213                                 clear_soft_out_cap(
 214                                         &test_vector.turbo_dec.op_flags);
 215                         }
 216
 217                         if (!flags_match(test_vector.turbo_dec.op_flags,
 218                                         cap->capability_flags))
 219                                 return TEST_FAILED;
 220                         if (nb_inputs > cap->num_buffers_src) {
 221                                 printf("Too many inputs defined: %u, max: %u\n",
 222                                         nb_inputs, cap->num_buffers_src);
 223                                 return TEST_FAILED;
 224                         }
 225                         if (nb_soft_outputs > cap->num_buffers_soft_out &&
 226                                         (test_vector.turbo_dec.op_flags &
 227                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 228                                 printf(
 229                                         "Too many soft outputs defined: %u, max: %u\n",
 230                                                 nb_soft_outputs,
 231                                                 cap->num_buffers_soft_out);
 232                                 return TEST_FAILED;
 233                         }
 234                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 235                                 printf(
 236                                         "Too many hard outputs defined: %u, max: %u\n",
 237                                                 nb_hard_outputs,
 238                                                 cap->num_buffers_hard_out);
 239                                 return TEST_FAILED;
 240                         }
 241                         if (intr_enabled && !(cap->capability_flags &
 242                                         RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
 243                                 printf(
 244                                         "Dequeue interrupts are not supported!\n");
 245                                 return TEST_FAILED;
 246                         }
 247
 248                         return TEST_SUCCESS;
 249                 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
 250                         const struct rte_bbdev_op_cap_turbo_enc *cap =
 251                                         &op_cap->cap.turbo_enc;
 252
 253                         if (!flags_match(test_vector.turbo_enc.op_flags,
 254                                         cap->capability_flags))
 255                                 return TEST_FAILED;
 256                         if (nb_inputs > cap->num_buffers_src) {
 257                                 printf("Too many inputs defined: %u, max: %u\n",
 258                                         nb_inputs, cap->num_buffers_src);
 259                                 return TEST_FAILED;
 260                         }
 261                         if (nb_hard_outputs > cap->num_buffers_dst) {
 262                                 printf(
 263                                         "Too many hard outputs defined: %u, max: %u\n",
 264                                         nb_hard_outputs, cap->num_buffers_dst);
 265                                 return TEST_FAILED;
 266                         }
 267                         if (intr_enabled && !(cap->capability_flags &
 268                                         RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
 269                                 printf(
 270                                         "Dequeue interrupts are not supported!\n");
 271                                 return TEST_FAILED;
 272                         }
 273
 274                         return TEST_SUCCESS;
 275                 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
 276                         const struct rte_bbdev_op_cap_ldpc_enc *cap =
 277                                         &op_cap->cap.ldpc_enc;
 278
 279                         if (!flags_match(test_vector.ldpc_enc.op_flags,
 280                                         cap->capability_flags)){
 281                                 printf("Flag Mismatch\n");
 282                                 return TEST_FAILED;
 283                         }
 284                         if (nb_inputs > cap->num_buffers_src) {
 285                                 printf("Too many inputs defined: %u, max: %u\n",
 286                                         nb_inputs, cap->num_buffers_src);
 287                                 return TEST_FAILED;
 288                         }
 289                         if (nb_hard_outputs > cap->num_buffers_dst) {
 290                                 printf(
 291                                         "Too many hard outputs defined: %u, max: %u\n",
 292                                         nb_hard_outputs, cap->num_buffers_dst);
 293                                 return TEST_FAILED;
 294                         }
 295                         if (intr_enabled && !(cap->capability_flags &
 296                                         RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
 297                                 printf(
 298                                         "Dequeue interrupts are not supported!\n");
 299                                 return TEST_FAILED;
 300                         }
 301
 302                         return TEST_SUCCESS;
 303                 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
 304                         const struct rte_bbdev_op_cap_ldpc_dec *cap =
 305                                         &op_cap->cap.ldpc_dec;
 306
 307                         if (!flags_match(test_vector.ldpc_dec.op_flags,
 308                                         cap->capability_flags)){
 309                                 printf("Flag Mismatch\n");
 310                                 return TEST_FAILED;
 311                         }
 312                         if (nb_inputs > cap->num_buffers_src) {
 313                                 printf("Too many inputs defined: %u, max: %u\n",
 314                                         nb_inputs, cap->num_buffers_src);
 315                                 return TEST_FAILED;
 316                         }
 317                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 318                                 printf(
 319                                         "Too many hard outputs defined: %u, max: %u\n",
 320                                         nb_hard_outputs,
 321                                         cap->num_buffers_hard_out);
 322                                 return TEST_FAILED;
 323                         }
 324                         if (nb_harq_inputs > cap->num_buffers_hard_out) {
 325                                 printf(
 326                                         "Too many HARQ inputs defined: %u, max: %u\n",
 327                                         nb_hard_outputs,
 328                                         cap->num_buffers_hard_out);
 329                                 return TEST_FAILED;
 330                         }
 331                         if (nb_harq_outputs > cap->num_buffers_hard_out) {
 332                                 printf(
 333                                         "Too many HARQ outputs defined: %u, max: %u\n",
 334                                         nb_hard_outputs,
 335                                         cap->num_buffers_hard_out);
 336                                 return TEST_FAILED;
 337                         }
 338                         if (intr_enabled && !(cap->capability_flags &
 339                                         RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
 340                                 printf(
 341                                         "Dequeue interrupts are not supported!\n");
 342                                 return TEST_FAILED;
 343                         }
 344
 345                         return TEST_SUCCESS;
 346                 }
 347         }
 348
 349         if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
 350                 return TEST_SUCCESS; /* Special case for NULL device */
 351
 352         return TEST_FAILED;
 353 }
 354
 355 /* calculates optimal mempool size not smaller than the val */
 356 static unsigned int
 357 optimal_mempool_size(unsigned int val)
 358 {
 359         return rte_align32pow2(val + 1) - 1;
 360 }
 361
 362 /* allocates mbuf mempool for inputs and outputs */
 363 static struct rte_mempool *
 364 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
 365                 int socket_id, unsigned int mbuf_pool_size,
 366                 const char *op_type_str)
 367 {
 368         unsigned int i;
 369         uint32_t max_seg_sz = 0;
 370         char pool_name[RTE_MEMPOOL_NAMESIZE];
 371
 372         /* find max input segment size */
 373         for (i = 0; i < entries->nb_segments; ++i)
 374                 if (entries->segments[i].length > max_seg_sz)
 375                         max_seg_sz = entries->segments[i].length;
 376
 377         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 378                         dev_id);
 379         return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
 380                         RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM,
 381                         (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
 382 }
 383
 384 static int
 385 create_mempools(struct active_device *ad, int socket_id,
 386                 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
 387 {
 388         struct rte_mempool *mp;
 389         unsigned int ops_pool_size, mbuf_pool_size = 0;
 390         char pool_name[RTE_MEMPOOL_NAMESIZE];
 391         const char *op_type_str;
 392         enum rte_bbdev_op_type op_type = org_op_type;
 393
 394         struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
 395         struct op_data_entries *hard_out =
 396                         &test_vector.entries[DATA_HARD_OUTPUT];
 397         struct op_data_entries *soft_out =
 398                         &test_vector.entries[DATA_SOFT_OUTPUT];
 399         struct op_data_entries *harq_in =
 400                         &test_vector.entries[DATA_HARQ_INPUT];
 401         struct op_data_entries *harq_out =
 402                         &test_vector.entries[DATA_HARQ_OUTPUT];
 403
 404         /* allocate ops mempool */
 405         ops_pool_size = optimal_mempool_size(RTE_MAX(
 406                         /* Ops used plus 1 reference op */
 407                         RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
 408                         /* Minimal cache size plus 1 reference op */
 409                         (unsigned int)(1.5 * rte_lcore_count() *
 410                                         OPS_CACHE_SIZE + 1)),
 411                         OPS_POOL_SIZE_MIN));
 412
 413         if (org_op_type == RTE_BBDEV_OP_NONE)
 414                 op_type = RTE_BBDEV_OP_TURBO_ENC;
 415
 416         op_type_str = rte_bbdev_op_type_str(op_type);
 417         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
 418
 419         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 420                         ad->dev_id);
 421         mp = rte_bbdev_op_pool_create(pool_name, op_type,
 422                         ops_pool_size, OPS_CACHE_SIZE, socket_id);
 423         TEST_ASSERT_NOT_NULL(mp,
 424                         "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
 425                         ops_pool_size,
 426                         ad->dev_id,
 427                         socket_id);
 428         ad->ops_mempool = mp;
 429
 430         /* Do not create inputs and outputs mbufs for BaseBand Null Device */
 431         if (org_op_type == RTE_BBDEV_OP_NONE)
 432                 return TEST_SUCCESS;
 433
 434         /* Inputs */
 435         mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments);
 436         mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in");
 437         TEST_ASSERT_NOT_NULL(mp,
 438                         "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
 439                         mbuf_pool_size,
 440                         ad->dev_id,
 441                         socket_id);
 442         ad->in_mbuf_pool = mp;
 443
 444         /* Hard outputs */
 445         mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 446                         hard_out->nb_segments);
 447         mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size,
 448                         "hard_out");
 449         TEST_ASSERT_NOT_NULL(mp,
 450                         "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
 451                         mbuf_pool_size,
 452                         ad->dev_id,
 453                         socket_id);
 454         ad->hard_out_mbuf_pool = mp;
 455
 456
 457         /* Soft outputs */
 458         if (soft_out->nb_segments > 0) {
 459                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 460                                 soft_out->nb_segments);
 461                 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
 462                                 mbuf_pool_size,
 463                                 "soft_out");
 464                 TEST_ASSERT_NOT_NULL(mp,
 465                                 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
 466                                 mbuf_pool_size,
 467                                 ad->dev_id,
 468                                 socket_id);
 469                 ad->soft_out_mbuf_pool = mp;
 470         }
 471
 472         /* HARQ inputs */
 473         if (harq_in->nb_segments > 0) {
 474                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 475                                 harq_in->nb_segments);
 476                 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
 477                                 mbuf_pool_size,
 478                                 "harq_in");
 479                 TEST_ASSERT_NOT_NULL(mp,
 480                                 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
 481                                 mbuf_pool_size,
 482                                 ad->dev_id,
 483                                 socket_id);
 484                 ad->harq_in_mbuf_pool = mp;
 485         }
 486
 487         /* HARQ outputs */
 488         if (harq_out->nb_segments > 0) {
 489                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 490                                 harq_out->nb_segments);
 491                 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
 492                                 mbuf_pool_size,
 493                                 "harq_out");
 494                 TEST_ASSERT_NOT_NULL(mp,
 495                                 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
 496                                 mbuf_pool_size,
 497                                 ad->dev_id,
 498                                 socket_id);
 499                 ad->harq_out_mbuf_pool = mp;
 500         }
 501
 502         return TEST_SUCCESS;
 503 }
 504
 505 static int
 506 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
 507                 struct test_bbdev_vector *vector)
 508 {
 509         int ret;
 510         unsigned int queue_id;
 511         struct rte_bbdev_queue_conf qconf;
 512         struct active_device *ad = &active_devs[nb_active_devs];
 513         unsigned int nb_queues;
 514         enum rte_bbdev_op_type op_type = vector->op_type;
 515
 516 /* Configure fpga lte fec with PF & VF values
 517  * if '-i' flag is set and using fpga device
 518  */
 519 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
 520         if ((get_init_device() == true) &&
 521                 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
 522                 struct fpga_lte_fec_conf conf;
 523                 unsigned int i;
 524
 525                 printf("Configure FPGA LTE FEC Driver %s with default values\n",
 526                                 info->drv.driver_name);
 527
 528                 /* clear default configuration before initialization */
 529                 memset(&conf, 0, sizeof(struct fpga_lte_fec_conf));
 530
 531                 /* Set PF mode :
 532                  * true if PF is used for data plane
 533                  * false for VFs
 534                  */
 535                 conf.pf_mode_en = true;
 536
 537                 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
 538                         /* Number of UL queues per VF (fpga supports 8 VFs) */
 539                         conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
 540                         /* Number of DL queues per VF (fpga supports 8 VFs) */
 541                         conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
 542                 }
 543
 544                 /* UL bandwidth. Needed for schedule algorithm */
 545                 conf.ul_bandwidth = UL_4G_BANDWIDTH;
 546                 /* DL bandwidth */
 547                 conf.dl_bandwidth = DL_4G_BANDWIDTH;
 548
 549                 /* UL & DL load Balance Factor to 64 */
 550                 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
 551                 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
 552
 553                 /**< FLR timeout value */
 554                 conf.flr_time_out = FLR_4G_TIMEOUT;
 555
 556                 /* setup FPGA PF with configuration information */
 557                 ret = fpga_lte_fec_configure(info->dev_name, &conf);
 558                 TEST_ASSERT_SUCCESS(ret,
 559                                 "Failed to configure 4G FPGA PF for bbdev %s",
 560                                 info->dev_name);
 561         }
 562 #endif
 563         nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
 564         nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
 565
 566         /* setup device */
 567         ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
 568         if (ret < 0) {
 569                 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
 570                                 dev_id, nb_queues, info->socket_id, ret);
 571                 return TEST_FAILED;
 572         }
 573
 574         /* configure interrupts if needed */
 575         if (intr_enabled) {
 576                 ret = rte_bbdev_intr_enable(dev_id);
 577                 if (ret < 0) {
 578                         printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
 579                                         ret);
 580                         return TEST_FAILED;
 581                 }
 582         }
 583
 584         /* setup device queues */
 585         qconf.socket = info->socket_id;
 586         qconf.queue_size = info->drv.default_queue_conf.queue_size;
 587         qconf.priority = 0;
 588         qconf.deferred_start = 0;
 589         qconf.op_type = op_type;
 590
 591         for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
 592                 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
 593                 if (ret != 0) {
 594                         printf(
 595                                         "Allocated all queues (id=%u) at prio%u on dev%u\n",
 596                                         queue_id, qconf.priority, dev_id);
 597                         qconf.priority++;
 598                         ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
 599                                         &qconf);
 600                 }
 601                 if (ret != 0) {
 602                         printf("All queues on dev %u allocated: %u\n",
 603                                         dev_id, queue_id);
 604                         break;
 605                 }
 606                 ad->queue_ids[queue_id] = queue_id;
 607         }
 608         TEST_ASSERT(queue_id != 0,
 609                         "ERROR Failed to configure any queues on dev %u",
 610                         dev_id);
 611         ad->nb_queues = queue_id;
 612
 613         set_avail_op(ad, op_type);
 614
 615         return TEST_SUCCESS;
 616 }
 617
 618 static int
 619 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
 620                 struct test_bbdev_vector *vector)
 621 {
 622         int ret;
 623
 624         active_devs[nb_active_devs].driver_name = info->drv.driver_name;
 625         active_devs[nb_active_devs].dev_id = dev_id;
 626
 627         ret = add_bbdev_dev(dev_id, info, vector);
 628         if (ret == TEST_SUCCESS)
 629                 ++nb_active_devs;
 630         return ret;
 631 }
 632
 633 static uint8_t
 634 populate_active_devices(void)
 635 {
 636         int ret;
 637         uint8_t dev_id;
 638         uint8_t nb_devs_added = 0;
 639         struct rte_bbdev_info info;
 640
 641         RTE_BBDEV_FOREACH(dev_id) {
 642                 rte_bbdev_info_get(dev_id, &info);
 643
 644                 if (check_dev_cap(&info)) {
 645                         printf(
 646                                 "Device %d (%s) does not support specified capabilities\n",
 647                                         dev_id, info.dev_name);
 648                         continue;
 649                 }
 650
 651                 ret = add_active_device(dev_id, &info, &test_vector);
 652                 if (ret != 0) {
 653                         printf("Adding active bbdev %s skipped\n",
 654                                         info.dev_name);
 655                         continue;
 656                 }
 657                 nb_devs_added++;
 658         }
 659
 660         return nb_devs_added;
 661 }
 662
 663 static int
 664 read_test_vector(void)
 665 {
 666         int ret;
 667
 668         memset(&test_vector, 0, sizeof(test_vector));
 669         printf("Test vector file = %s\n", get_vector_filename());
 670         ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
 671         TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
 672                         get_vector_filename());
 673
 674         return TEST_SUCCESS;
 675 }
 676
 677 static int
 678 testsuite_setup(void)
 679 {
 680         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 681
 682         if (populate_active_devices() == 0) {
 683                 printf("No suitable devices found!\n");
 684                 return TEST_SKIPPED;
 685         }
 686
 687         return TEST_SUCCESS;
 688 }
 689
 690 static int
 691 interrupt_testsuite_setup(void)
 692 {
 693         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 694
 695         /* Enable interrupts */
 696         intr_enabled = true;
 697
 698         /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
 699         if (populate_active_devices() == 0 ||
 700                         test_vector.op_type == RTE_BBDEV_OP_NONE) {
 701                 intr_enabled = false;
 702                 printf("No suitable devices found!\n");
 703                 return TEST_SKIPPED;
 704         }
 705
 706         return TEST_SUCCESS;
 707 }
 708
 709 static void
 710 testsuite_teardown(void)
 711 {
 712         uint8_t dev_id;
 713
 714         /* Unconfigure devices */
 715         RTE_BBDEV_FOREACH(dev_id)
 716                 rte_bbdev_close(dev_id);
 717
 718         /* Clear active devices structs. */
 719         memset(active_devs, 0, sizeof(active_devs));
 720         nb_active_devs = 0;
 721 }
 722
 723 static int
 724 ut_setup(void)
 725 {
 726         uint8_t i, dev_id;
 727
 728         for (i = 0; i < nb_active_devs; i++) {
 729                 dev_id = active_devs[i].dev_id;
 730                 /* reset bbdev stats */
 731                 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
 732                                 "Failed to reset stats of bbdev %u", dev_id);
 733                 /* start the device */
 734                 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
 735                                 "Failed to start bbdev %u", dev_id);
 736         }
 737
 738         return TEST_SUCCESS;
 739 }
 740
 741 static void
 742 ut_teardown(void)
 743 {
 744         uint8_t i, dev_id;
 745         struct rte_bbdev_stats stats;
 746
 747         for (i = 0; i < nb_active_devs; i++) {
 748                 dev_id = active_devs[i].dev_id;
 749                 /* read stats and print */
 750                 rte_bbdev_stats_get(dev_id, &stats);
 751                 /* Stop the device */
 752                 rte_bbdev_stop(dev_id);
 753         }
 754 }
 755
 756 static int
 757 init_op_data_objs(struct rte_bbdev_op_data *bufs,
 758                 struct op_data_entries *ref_entries,
 759                 struct rte_mempool *mbuf_pool, const uint16_t n,
 760                 enum op_data_type op_type, uint16_t min_alignment)
 761 {
 762         int ret;
 763         unsigned int i, j;
 764         bool large_input = false;
 765
 766         for (i = 0; i < n; ++i) {
 767                 char *data;
 768                 struct op_data_buf *seg = &ref_entries->segments[0];
 769                 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
 770                 TEST_ASSERT_NOT_NULL(m_head,
 771                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 772                                 op_type, n * ref_entries->nb_segments,
 773                                 mbuf_pool->size);
 774
 775                 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
 776                         /*
 777                          * Special case when DPDK mbuf cannot handle
 778                          * the required input size
 779                          */
 780                         printf("Warning: Larger input size than DPDK mbuf %d\n",
 781                                         seg->length);
 782                         large_input = true;
 783                 }
 784                 bufs[i].data = m_head;
 785                 bufs[i].offset = 0;
 786                 bufs[i].length = 0;
 787
 788                 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
 789                         if ((op_type == DATA_INPUT) && large_input) {
 790                                 /* Allocate a fake overused mbuf */
 791                                 data = rte_malloc(NULL, seg->length, 0);
 792                                 memcpy(data, seg->addr, seg->length);
 793                                 m_head->buf_addr = data;
 794                                 m_head->buf_iova = rte_malloc_virt2iova(data);
 795                                 m_head->data_off = 0;
 796                                 m_head->data_len = seg->length;
 797                         } else {
 798                                 data = rte_pktmbuf_append(m_head, seg->length);
 799                                 TEST_ASSERT_NOT_NULL(data,
 800                                         "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 801                                         seg->length, op_type);
 802
 803                                 TEST_ASSERT(data == RTE_PTR_ALIGN(
 804                                                 data, min_alignment),
 805                                         "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 806                                         data, min_alignment);
 807                                 rte_memcpy(data, seg->addr, seg->length);
 808                         }
 809
 810                         bufs[i].length += seg->length;
 811
 812                         for (j = 1; j < ref_entries->nb_segments; ++j) {
 813                                 struct rte_mbuf *m_tail =
 814                                                 rte_pktmbuf_alloc(mbuf_pool);
 815                                 TEST_ASSERT_NOT_NULL(m_tail,
 816                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 817                                                 op_type,
 818                                                 n * ref_entries->nb_segments,
 819                                                 mbuf_pool->size);
 820                                 seg += 1;
 821
 822                                 data = rte_pktmbuf_append(m_tail, seg->length);
 823                                 TEST_ASSERT_NOT_NULL(data,
 824                                                 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 825                                                 seg->length, op_type);
 826
 827                                 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
 828                                                 min_alignment),
 829                                                 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 830                                                 data, min_alignment);
 831                                 rte_memcpy(data, seg->addr, seg->length);
 832                                 bufs[i].length += seg->length;
 833
 834                                 ret = rte_pktmbuf_chain(m_head, m_tail);
 835                                 TEST_ASSERT_SUCCESS(ret,
 836                                                 "Couldn't chain mbufs from %d data type mbuf pool",
 837                                                 op_type);
 838                         }
 839                 } else {
 840
 841                         /* allocate chained-mbuf for output buffer */
 842                         for (j = 1; j < ref_entries->nb_segments; ++j) {
 843                                 struct rte_mbuf *m_tail =
 844                                                 rte_pktmbuf_alloc(mbuf_pool);
 845                                 TEST_ASSERT_NOT_NULL(m_tail,
 846                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 847                                                 op_type,
 848                                                 n * ref_entries->nb_segments,
 849                                                 mbuf_pool->size);
 850
 851                                 ret = rte_pktmbuf_chain(m_head, m_tail);
 852                                 TEST_ASSERT_SUCCESS(ret,
 853                                                 "Couldn't chain mbufs from %d data type mbuf pool",
 854                                                 op_type);
 855                         }
 856                 }
 857         }
 858
 859         return 0;
 860 }
 861
 862 static int
 863 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
 864                 const int socket)
 865 {
 866         int i;
 867
 868         *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
 869         if (*buffers == NULL) {
 870                 printf("WARNING: Failed to allocate op_data on socket %d\n",
 871                                 socket);
 872                 /* try to allocate memory on other detected sockets */
 873                 for (i = 0; i < socket; i++) {
 874                         *buffers = rte_zmalloc_socket(NULL, len, 0, i);
 875                         if (*buffers != NULL)
 876                                 break;
 877                 }
 878         }
 879
 880         return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
 881 }
 882
 883 static void
 884 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
 885                 const uint16_t n, const int8_t max_llr_modulus)
 886 {
 887         uint16_t i, byte_idx;
 888
 889         for (i = 0; i < n; ++i) {
 890                 struct rte_mbuf *m = input_ops[i].data;
 891                 while (m != NULL) {
 892                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
 893                                         input_ops[i].offset);
 894                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
 895                                         ++byte_idx)
 896                                 llr[byte_idx] = round((double)max_llr_modulus *
 897                                                 llr[byte_idx] / INT8_MAX);
 898
 899                         m = m->next;
 900                 }
 901         }
 902 }
 903
 904 static void
 905 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
 906                 const uint16_t n, const int8_t llr_size,
 907                 const int8_t llr_decimals)
 908 {
 909         if (input_ops == NULL)
 910                 return;
 911
 912         uint16_t i, byte_idx;
 913
 914         int16_t llr_max, llr_min, llr_tmp;
 915         llr_max = (1 << (llr_size - 1)) - 1;
 916         llr_min = -llr_max;
 917         for (i = 0; i < n; ++i) {
 918                 struct rte_mbuf *m = input_ops[i].data;
 919                 while (m != NULL) {
 920                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
 921                                         input_ops[i].offset);
 922                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
 923                                         ++byte_idx) {
 924
 925                                 llr_tmp = llr[byte_idx];
 926                                 if (llr_decimals == 2)
 927                                         llr_tmp *= 2;
 928                                 else if (llr_decimals == 0)
 929                                         llr_tmp /= 2;
 930                                 llr_tmp = RTE_MIN(llr_max,
 931                                                 RTE_MAX(llr_min, llr_tmp));
 932                                 llr[byte_idx] = (int8_t) llr_tmp;
 933                         }
 934
 935                         m = m->next;
 936                 }
 937         }
 938 }
 939
 940
 941
 942 static int
 943 fill_queue_buffers(struct test_op_params *op_params,
 944                 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
 945                 struct rte_mempool *soft_out_mp,
 946                 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
 947                 uint16_t queue_id,
 948                 const struct rte_bbdev_op_cap *capabilities,
 949                 uint16_t min_alignment, const int socket_id)
 950 {
 951         int ret;
 952         enum op_data_type type;
 953         const uint16_t n = op_params->num_to_process;
 954
 955         struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
 956                 in_mp,
 957                 soft_out_mp,
 958                 hard_out_mp,
 959                 harq_in_mp,
 960                 harq_out_mp,
 961         };
 962
 963         struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
 964                 &op_params->q_bufs[socket_id][queue_id].inputs,
 965                 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
 966                 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
 967                 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
 968                 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
 969         };
 970
 971         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
 972                 struct op_data_entries *ref_entries =
 973                                 &test_vector.entries[type];
 974                 if (ref_entries->nb_segments == 0)
 975                         continue;
 976
 977                 ret = allocate_buffers_on_socket(queue_ops[type],
 978                                 n * sizeof(struct rte_bbdev_op_data),
 979                                 socket_id);
 980                 TEST_ASSERT_SUCCESS(ret,
 981                                 "Couldn't allocate memory for rte_bbdev_op_data structs");
 982
 983                 ret = init_op_data_objs(*queue_ops[type], ref_entries,
 984                                 mbuf_pools[type], n, type, min_alignment);
 985                 TEST_ASSERT_SUCCESS(ret,
 986                                 "Couldn't init rte_bbdev_op_data structs");
 987         }
 988
 989         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
 990                 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
 991                         capabilities->cap.turbo_dec.max_llr_modulus);
 992
 993         if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
 994                 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
 995                         capabilities->cap.ldpc_dec.llr_size,
 996                         capabilities->cap.ldpc_dec.llr_decimals);
 997                 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
 998                                 capabilities->cap.ldpc_dec.llr_size,
 999                                 capabilities->cap.ldpc_dec.llr_decimals);
1000         }
1001
1002         return 0;
1003 }
1004
1005 static void
1006 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1007 {
1008         unsigned int i, j;
1009
1010         rte_mempool_free(ad->ops_mempool);
1011         rte_mempool_free(ad->in_mbuf_pool);
1012         rte_mempool_free(ad->hard_out_mbuf_pool);
1013         rte_mempool_free(ad->soft_out_mbuf_pool);
1014         rte_mempool_free(ad->harq_in_mbuf_pool);
1015         rte_mempool_free(ad->harq_out_mbuf_pool);
1016
1017         for (i = 0; i < rte_lcore_count(); ++i) {
1018                 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1019                         rte_free(op_params->q_bufs[j][i].inputs);
1020                         rte_free(op_params->q_bufs[j][i].hard_outputs);
1021                         rte_free(op_params->q_bufs[j][i].soft_outputs);
1022                         rte_free(op_params->q_bufs[j][i].harq_inputs);
1023                         rte_free(op_params->q_bufs[j][i].harq_outputs);
1024                 }
1025         }
1026 }
1027
1028 static void
1029 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1030                 unsigned int start_idx,
1031                 struct rte_bbdev_op_data *inputs,
1032                 struct rte_bbdev_op_data *hard_outputs,
1033                 struct rte_bbdev_op_data *soft_outputs,
1034                 struct rte_bbdev_dec_op *ref_op)
1035 {
1036         unsigned int i;
1037         struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1038
1039         for (i = 0; i < n; ++i) {
1040                 if (turbo_dec->code_block_mode == 0) {
1041                         ops[i]->turbo_dec.tb_params.ea =
1042                                         turbo_dec->tb_params.ea;
1043                         ops[i]->turbo_dec.tb_params.eb =
1044                                         turbo_dec->tb_params.eb;
1045                         ops[i]->turbo_dec.tb_params.k_pos =
1046                                         turbo_dec->tb_params.k_pos;
1047                         ops[i]->turbo_dec.tb_params.k_neg =
1048                                         turbo_dec->tb_params.k_neg;
1049                         ops[i]->turbo_dec.tb_params.c =
1050                                         turbo_dec->tb_params.c;
1051                         ops[i]->turbo_dec.tb_params.c_neg =
1052                                         turbo_dec->tb_params.c_neg;
1053                         ops[i]->turbo_dec.tb_params.cab =
1054                                         turbo_dec->tb_params.cab;
1055                         ops[i]->turbo_dec.tb_params.r =
1056                                         turbo_dec->tb_params.r;
1057                 } else {
1058                         ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1059                         ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1060                 }
1061
1062                 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1063                 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1064                 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1065                 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1066                 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1067                 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1068                 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1069
1070                 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1071                 ops[i]->turbo_dec.input = inputs[start_idx + i];
1072                 if (soft_outputs != NULL)
1073                         ops[i]->turbo_dec.soft_output =
1074                                 soft_outputs[start_idx + i];
1075         }
1076 }
1077
1078 static void
1079 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1080                 unsigned int start_idx,
1081                 struct rte_bbdev_op_data *inputs,
1082                 struct rte_bbdev_op_data *outputs,
1083                 struct rte_bbdev_enc_op *ref_op)
1084 {
1085         unsigned int i;
1086         struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1087         for (i = 0; i < n; ++i) {
1088                 if (turbo_enc->code_block_mode == 0) {
1089                         ops[i]->turbo_enc.tb_params.ea =
1090                                         turbo_enc->tb_params.ea;
1091                         ops[i]->turbo_enc.tb_params.eb =
1092                                         turbo_enc->tb_params.eb;
1093                         ops[i]->turbo_enc.tb_params.k_pos =
1094                                         turbo_enc->tb_params.k_pos;
1095                         ops[i]->turbo_enc.tb_params.k_neg =
1096                                         turbo_enc->tb_params.k_neg;
1097                         ops[i]->turbo_enc.tb_params.c =
1098                                         turbo_enc->tb_params.c;
1099                         ops[i]->turbo_enc.tb_params.c_neg =
1100                                         turbo_enc->tb_params.c_neg;
1101                         ops[i]->turbo_enc.tb_params.cab =
1102                                         turbo_enc->tb_params.cab;
1103                         ops[i]->turbo_enc.tb_params.ncb_pos =
1104                                         turbo_enc->tb_params.ncb_pos;
1105                         ops[i]->turbo_enc.tb_params.ncb_neg =
1106                                         turbo_enc->tb_params.ncb_neg;
1107                         ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1108                 } else {
1109                         ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1110                         ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1111                         ops[i]->turbo_enc.cb_params.ncb =
1112                                         turbo_enc->cb_params.ncb;
1113                 }
1114                 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1115                 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1116                 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1117
1118                 ops[i]->turbo_enc.output = outputs[start_idx + i];
1119                 ops[i]->turbo_enc.input = inputs[start_idx + i];
1120         }
1121 }
1122
1123 static void
1124 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1125                 unsigned int start_idx,
1126                 struct rte_bbdev_op_data *inputs,
1127                 struct rte_bbdev_op_data *hard_outputs,
1128                 struct rte_bbdev_op_data *soft_outputs,
1129                 struct rte_bbdev_op_data *harq_inputs,
1130                 struct rte_bbdev_op_data *harq_outputs,
1131                 struct rte_bbdev_dec_op *ref_op)
1132 {
1133         unsigned int i;
1134         struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1135
1136         for (i = 0; i < n; ++i) {
1137                 if (ldpc_dec->code_block_mode == 0) {
1138                         ops[i]->ldpc_dec.tb_params.ea =
1139                                         ldpc_dec->tb_params.ea;
1140                         ops[i]->ldpc_dec.tb_params.eb =
1141                                         ldpc_dec->tb_params.eb;
1142                         ops[i]->ldpc_dec.tb_params.c =
1143                                         ldpc_dec->tb_params.c;
1144                         ops[i]->ldpc_dec.tb_params.cab =
1145                                         ldpc_dec->tb_params.cab;
1146                         ops[i]->ldpc_dec.tb_params.r =
1147                                         ldpc_dec->tb_params.r;
1148                 } else {
1149                         ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1150                 }
1151
1152                 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1153                 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1154                 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1155                 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1156                 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1157                 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1158                 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1159                 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1160                 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1161
1162                 ops[i]->ldpc_dec.hard_output = hard_outputs[start_idx + i];
1163                 ops[i]->ldpc_dec.input = inputs[start_idx + i];
1164                 if (soft_outputs != NULL)
1165                         ops[i]->ldpc_dec.soft_output =
1166                                 soft_outputs[start_idx + i];
1167                 if (harq_inputs != NULL)
1168                         ops[i]->ldpc_dec.harq_combined_input =
1169                                         harq_inputs[start_idx + i];
1170                 if (harq_outputs != NULL)
1171                         ops[i]->ldpc_dec.harq_combined_output =
1172                                 harq_outputs[start_idx + i];
1173         }
1174 }
1175
1176
1177 static void
1178 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1179                 unsigned int start_idx,
1180                 struct rte_bbdev_op_data *inputs,
1181                 struct rte_bbdev_op_data *outputs,
1182                 struct rte_bbdev_enc_op *ref_op)
1183 {
1184         unsigned int i;
1185         struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1186         for (i = 0; i < n; ++i) {
1187                 if (ldpc_enc->code_block_mode == 0) {
1188                         ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1189                         ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1190                         ops[i]->ldpc_enc.tb_params.cab =
1191                                         ldpc_enc->tb_params.cab;
1192                         ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1193                         ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1194                 } else {
1195                         ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1196                 }
1197                 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1198                 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1199                 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1200                 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1201                 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1202                 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1203                 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1204                 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1205                 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1206                 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1207         }
1208 }
1209
1210 static int
1211 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1212                 unsigned int order_idx, const int expected_status)
1213 {
1214         TEST_ASSERT(op->status == expected_status,
1215                         "op_status (%d) != expected_status (%d)",
1216                         op->status, expected_status);
1217
1218         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1219                         "Ordering error, expected %p, got %p",
1220                         (void *)(uintptr_t)order_idx, op->opaque_data);
1221
1222         return TEST_SUCCESS;
1223 }
1224
1225 static int
1226 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1227                 unsigned int order_idx, const int expected_status)
1228 {
1229         TEST_ASSERT(op->status == expected_status,
1230                         "op_status (%d) != expected_status (%d)",
1231                         op->status, expected_status);
1232
1233         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1234                         "Ordering error, expected %p, got %p",
1235                         (void *)(uintptr_t)order_idx, op->opaque_data);
1236
1237         return TEST_SUCCESS;
1238 }
1239
1240 static inline int
1241 validate_op_chain(struct rte_bbdev_op_data *op,
1242                 struct op_data_entries *orig_op)
1243 {
1244         uint8_t i;
1245         struct rte_mbuf *m = op->data;
1246         uint8_t nb_dst_segments = orig_op->nb_segments;
1247         uint32_t total_data_size = 0;
1248
1249         TEST_ASSERT(nb_dst_segments == m->nb_segs,
1250                         "Number of segments differ in original (%u) and filled (%u) op",
1251                         nb_dst_segments, m->nb_segs);
1252
1253         /* Validate each mbuf segment length */
1254         for (i = 0; i < nb_dst_segments; ++i) {
1255                 /* Apply offset to the first mbuf segment */
1256                 uint16_t offset = (i == 0) ? op->offset : 0;
1257                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1258                 total_data_size += orig_op->segments[i].length;
1259
1260                 TEST_ASSERT(orig_op->segments[i].length == data_len,
1261                                 "Length of segment differ in original (%u) and filled (%u) op",
1262                                 orig_op->segments[i].length, data_len);
1263                 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1264                                 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1265                                 data_len,
1266                                 "Output buffers (CB=%u) are not equal", i);
1267                 m = m->next;
1268         }
1269
1270         /* Validate total mbuf pkt length */
1271         uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1272         TEST_ASSERT(total_data_size == pkt_len,
1273                         "Length of data differ in original (%u) and filled (%u) op",
1274                         total_data_size, pkt_len);
1275
1276         return TEST_SUCCESS;
1277 }
1278
1279 static int
1280 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1281                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1282 {
1283         unsigned int i;
1284         int ret;
1285         struct op_data_entries *hard_data_orig =
1286                         &test_vector.entries[DATA_HARD_OUTPUT];
1287         struct op_data_entries *soft_data_orig =
1288                         &test_vector.entries[DATA_SOFT_OUTPUT];
1289         struct rte_bbdev_op_turbo_dec *ops_td;
1290         struct rte_bbdev_op_data *hard_output;
1291         struct rte_bbdev_op_data *soft_output;
1292         struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
1293
1294         for (i = 0; i < n; ++i) {
1295                 ops_td = &ops[i]->turbo_dec;
1296                 hard_output = &ops_td->hard_output;
1297                 soft_output = &ops_td->soft_output;
1298
1299                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
1300                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
1301                                         "Returned iter_count (%d) > expected iter_count (%d)",
1302                                         ops_td->iter_count, ref_td->iter_count);
1303                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
1304                 TEST_ASSERT_SUCCESS(ret,
1305                                 "Checking status and ordering for decoder failed");
1306
1307                 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
1308                                 hard_data_orig),
1309                                 "Hard output buffers (CB=%u) are not equal",
1310                                 i);
1311
1312                 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
1313                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
1314                                         soft_data_orig),
1315                                         "Soft output buffers (CB=%u) are not equal",
1316                                         i);
1317         }
1318
1319         return TEST_SUCCESS;
1320 }
1321
1322
1323 static int
1324 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1325                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1326 {
1327         unsigned int i;
1328         int ret;
1329         struct op_data_entries *hard_data_orig =
1330                         &test_vector.entries[DATA_HARD_OUTPUT];
1331         struct op_data_entries *soft_data_orig =
1332                         &test_vector.entries[DATA_SOFT_OUTPUT];
1333         struct op_data_entries *harq_data_orig =
1334                                 &test_vector.entries[DATA_HARQ_OUTPUT];
1335         struct rte_bbdev_op_ldpc_dec *ops_td;
1336         struct rte_bbdev_op_data *hard_output;
1337         struct rte_bbdev_op_data *harq_output;
1338         struct rte_bbdev_op_data *soft_output;
1339         struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
1340
1341         for (i = 0; i < n; ++i) {
1342                 ops_td = &ops[i]->ldpc_dec;
1343                 hard_output = &ops_td->hard_output;
1344                 harq_output = &ops_td->harq_combined_output;
1345                 soft_output = &ops_td->soft_output;
1346
1347                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
1348                 TEST_ASSERT_SUCCESS(ret,
1349                                 "Checking status and ordering for decoder failed");
1350                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
1351                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
1352                                         "Returned iter_count (%d) > expected iter_count (%d)",
1353                                         ops_td->iter_count, ref_td->iter_count);
1354                 /* We can ignore data when the decoding failed to converge */
1355                 if ((ops[i]->status &  (1 << RTE_BBDEV_SYNDROME_ERROR)) == 0)
1356                         TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
1357                                         hard_data_orig),
1358                                         "Hard output buffers (CB=%u) are not equal",
1359                                         i);
1360
1361                 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
1362                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
1363                                         soft_data_orig),
1364                                         "Soft output buffers (CB=%u) are not equal",
1365                                         i);
1366                 if (ref_op->ldpc_dec.op_flags &
1367                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
1368                         ldpc_input_llr_scaling(harq_output, 1, 8, 0);
1369                         TEST_ASSERT_SUCCESS(validate_op_chain(harq_output,
1370                                         harq_data_orig),
1371                                         "HARQ output buffers (CB=%u) are not equal",
1372                                         i);
1373                 }
1374         }
1375
1376         return TEST_SUCCESS;
1377 }
1378
1379
1380 static int
1381 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
1382                 struct rte_bbdev_enc_op *ref_op)
1383 {
1384         unsigned int i;
1385         int ret;
1386         struct op_data_entries *hard_data_orig =
1387                         &test_vector.entries[DATA_HARD_OUTPUT];
1388
1389         for (i = 0; i < n; ++i) {
1390                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
1391                 TEST_ASSERT_SUCCESS(ret,
1392                                 "Checking status and ordering for encoder failed");
1393                 TEST_ASSERT_SUCCESS(validate_op_chain(
1394                                 &ops[i]->turbo_enc.output,
1395                                 hard_data_orig),
1396                                 "Output buffers (CB=%u) are not equal",
1397                                 i);
1398         }
1399
1400         return TEST_SUCCESS;
1401 }
1402
1403 static int
1404 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
1405                 struct rte_bbdev_enc_op *ref_op)
1406 {
1407         unsigned int i;
1408         int ret;
1409         struct op_data_entries *hard_data_orig =
1410                         &test_vector.entries[DATA_HARD_OUTPUT];
1411
1412         for (i = 0; i < n; ++i) {
1413                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
1414                 TEST_ASSERT_SUCCESS(ret,
1415                                 "Checking status and ordering for encoder failed");
1416                 TEST_ASSERT_SUCCESS(validate_op_chain(
1417                                 &ops[i]->ldpc_enc.output,
1418                                 hard_data_orig),
1419                                 "Output buffers (CB=%u) are not equal",
1420                                 i);
1421         }
1422
1423         return TEST_SUCCESS;
1424 }
1425
1426 static void
1427 create_reference_dec_op(struct rte_bbdev_dec_op *op)
1428 {
1429         unsigned int i;
1430         struct op_data_entries *entry;
1431
1432         op->turbo_dec = test_vector.turbo_dec;
1433         entry = &test_vector.entries[DATA_INPUT];
1434         for (i = 0; i < entry->nb_segments; ++i)
1435                 op->turbo_dec.input.length +=
1436                                 entry->segments[i].length;
1437 }
1438
1439 static void
1440 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
1441 {
1442         unsigned int i;
1443         struct op_data_entries *entry;
1444
1445         op->ldpc_dec = test_vector.ldpc_dec;
1446         entry = &test_vector.entries[DATA_INPUT];
1447         for (i = 0; i < entry->nb_segments; ++i)
1448                 op->ldpc_dec.input.length +=
1449                                 entry->segments[i].length;
1450         if (test_vector.ldpc_dec.op_flags &
1451                         RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
1452                 entry = &test_vector.entries[DATA_HARQ_INPUT];
1453                 for (i = 0; i < entry->nb_segments; ++i)
1454                         op->ldpc_dec.harq_combined_input.length +=
1455                                 entry->segments[i].length;
1456         }
1457 }
1458
1459
1460 static void
1461 create_reference_enc_op(struct rte_bbdev_enc_op *op)
1462 {
1463         unsigned int i;
1464         struct op_data_entries *entry;
1465
1466         op->turbo_enc = test_vector.turbo_enc;
1467         entry = &test_vector.entries[DATA_INPUT];
1468         for (i = 0; i < entry->nb_segments; ++i)
1469                 op->turbo_enc.input.length +=
1470                                 entry->segments[i].length;
1471 }
1472
1473 static void
1474 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
1475 {
1476         unsigned int i;
1477         struct op_data_entries *entry;
1478
1479         op->ldpc_enc = test_vector.ldpc_enc;
1480         entry = &test_vector.entries[DATA_INPUT];
1481         for (i = 0; i < entry->nb_segments; ++i)
1482                 op->ldpc_enc.input.length +=
1483                                 entry->segments[i].length;
1484 }
1485
1486 static uint32_t
1487 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
1488 {
1489         uint8_t i;
1490         uint32_t c, r, tb_size = 0;
1491
1492         if (op->turbo_dec.code_block_mode) {
1493                 tb_size = op->turbo_dec.tb_params.k_neg;
1494         } else {
1495                 c = op->turbo_dec.tb_params.c;
1496                 r = op->turbo_dec.tb_params.r;
1497                 for (i = 0; i < c-r; i++)
1498                         tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
1499                                 op->turbo_dec.tb_params.k_neg :
1500                                 op->turbo_dec.tb_params.k_pos;
1501         }
1502         return tb_size;
1503 }
1504
1505 static uint32_t
1506 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
1507 {
1508         uint8_t i;
1509         uint32_t c, r, tb_size = 0;
1510         uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
1511
1512         if (op->ldpc_dec.code_block_mode) {
1513                 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
1514         } else {
1515                 c = op->ldpc_dec.tb_params.c;
1516                 r = op->ldpc_dec.tb_params.r;
1517                 for (i = 0; i < c-r; i++)
1518                         tb_size += sys_cols * op->ldpc_dec.z_c
1519                                         - op->ldpc_dec.n_filler;
1520         }
1521         return tb_size;
1522 }
1523
1524 static uint32_t
1525 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
1526 {
1527         uint8_t i;
1528         uint32_t c, r, tb_size = 0;
1529
1530         if (op->turbo_enc.code_block_mode) {
1531                 tb_size = op->turbo_enc.tb_params.k_neg;
1532         } else {
1533                 c = op->turbo_enc.tb_params.c;
1534                 r = op->turbo_enc.tb_params.r;
1535                 for (i = 0; i < c-r; i++)
1536                         tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
1537                                 op->turbo_enc.tb_params.k_neg :
1538                                 op->turbo_enc.tb_params.k_pos;
1539         }
1540         return tb_size;
1541 }
1542
1543 static uint32_t
1544 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
1545 {
1546         uint8_t i;
1547         uint32_t c, r, tb_size = 0;
1548         uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
1549
1550         if (op->turbo_enc.code_block_mode) {
1551                 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
1552         } else {
1553                 c = op->turbo_enc.tb_params.c;
1554                 r = op->turbo_enc.tb_params.r;
1555                 for (i = 0; i < c-r; i++)
1556                         tb_size += sys_cols * op->ldpc_enc.z_c
1557                                         - op->ldpc_enc.n_filler;
1558         }
1559         return tb_size;
1560 }
1561
1562
1563 static int
1564 init_test_op_params(struct test_op_params *op_params,
1565                 enum rte_bbdev_op_type op_type, const int expected_status,
1566                 const int vector_mask, struct rte_mempool *ops_mp,
1567                 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
1568 {
1569         int ret = 0;
1570         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
1571                         op_type == RTE_BBDEV_OP_LDPC_DEC)
1572                 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
1573                                 &op_params->ref_dec_op, 1);
1574         else
1575                 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
1576                                 &op_params->ref_enc_op, 1);
1577
1578         TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
1579
1580         op_params->mp = ops_mp;
1581         op_params->burst_sz = burst_sz;
1582         op_params->num_to_process = num_to_process;
1583         op_params->num_lcores = num_lcores;
1584         op_params->vector_mask = vector_mask;
1585         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
1586                         op_type == RTE_BBDEV_OP_LDPC_DEC)
1587                 op_params->ref_dec_op->status = expected_status;
1588         else if (op_type == RTE_BBDEV_OP_TURBO_ENC
1589                         || op_type == RTE_BBDEV_OP_LDPC_ENC)
1590                 op_params->ref_enc_op->status = expected_status;
1591         return 0;
1592 }
1593
1594 static int
1595 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
1596                 struct test_op_params *op_params)
1597 {
1598         int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
1599         unsigned int i;
1600         struct active_device *ad;
1601         unsigned int burst_sz = get_burst_sz();
1602         enum rte_bbdev_op_type op_type = test_vector.op_type;
1603         const struct rte_bbdev_op_cap *capabilities = NULL;
1604
1605         ad = &active_devs[dev_id];
1606
1607         /* Check if device supports op_type */
1608         if (!is_avail_op(ad, test_vector.op_type))
1609                 return TEST_SUCCESS;
1610
1611         struct rte_bbdev_info info;
1612         rte_bbdev_info_get(ad->dev_id, &info);
1613         socket_id = GET_SOCKET(info.socket_id);
1614
1615         f_ret = create_mempools(ad, socket_id, op_type,
1616                         get_num_ops());
1617         if (f_ret != TEST_SUCCESS) {
1618                 printf("Couldn't create mempools");
1619                 goto fail;
1620         }
1621         if (op_type == RTE_BBDEV_OP_NONE)
1622                 op_type = RTE_BBDEV_OP_TURBO_ENC;
1623
1624         f_ret = init_test_op_params(op_params, test_vector.op_type,
1625                         test_vector.expected_status,
1626                         test_vector.mask,
1627                         ad->ops_mempool,
1628                         burst_sz,
1629                         get_num_ops(),
1630                         get_num_lcores());
1631         if (f_ret != TEST_SUCCESS) {
1632                 printf("Couldn't init test op params");
1633                 goto fail;
1634         }
1635
1636
1637         /* Find capabilities */
1638         const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
1639         for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
1640                 if (cap->type == test_vector.op_type) {
1641                         capabilities = cap;
1642                         break;
1643                 }
1644                 cap++;
1645         }
1646         TEST_ASSERT_NOT_NULL(capabilities,
1647                         "Couldn't find capabilities");
1648
1649         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
1650                 create_reference_dec_op(op_params->ref_dec_op);
1651         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1652                 create_reference_enc_op(op_params->ref_enc_op);
1653         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
1654                 create_reference_ldpc_enc_op(op_params->ref_enc_op);
1655         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
1656                 create_reference_ldpc_dec_op(op_params->ref_dec_op);
1657
1658         for (i = 0; i < ad->nb_queues; ++i) {
1659                 f_ret = fill_queue_buffers(op_params,
1660                                 ad->in_mbuf_pool,
1661                                 ad->hard_out_mbuf_pool,
1662                                 ad->soft_out_mbuf_pool,
1663                                 ad->harq_in_mbuf_pool,
1664                                 ad->harq_out_mbuf_pool,
1665                                 ad->queue_ids[i],
1666                                 capabilities,
1667                                 info.drv.min_alignment,
1668                                 socket_id);
1669                 if (f_ret != TEST_SUCCESS) {
1670                         printf("Couldn't init queue buffers");
1671                         goto fail;
1672                 }
1673         }
1674
1675         /* Run test case function */
1676         t_ret = test_case_func(ad, op_params);
1677
1678         /* Free active device resources and return */
1679         free_buffers(ad, op_params);
1680         return t_ret;
1681
1682 fail:
1683         free_buffers(ad, op_params);
1684         return TEST_FAILED;
1685 }
1686
1687 /* Run given test function per active device per supported op type
1688  * per burst size.
1689  */
1690 static int
1691 run_test_case(test_case_function *test_case_func)
1692 {
1693         int ret = 0;
1694         uint8_t dev;
1695
1696         /* Alloc op_params */
1697         struct test_op_params *op_params = rte_zmalloc(NULL,
1698                         sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
1699         TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
1700                         RTE_ALIGN(sizeof(struct test_op_params),
1701                                 RTE_CACHE_LINE_SIZE));
1702
1703         /* For each device run test case function */
1704         for (dev = 0; dev < nb_active_devs; ++dev)
1705                 ret |= run_test_case_on_device(test_case_func, dev, op_params);
1706
1707         rte_free(op_params);
1708
1709         return ret;
1710 }
1711
1712 static void
1713 dequeue_event_callback(uint16_t dev_id,
1714                 enum rte_bbdev_event_type event, void *cb_arg,
1715                 void *ret_param)
1716 {
1717         int ret;
1718         uint16_t i;
1719         uint64_t total_time;
1720         uint16_t deq, burst_sz, num_ops;
1721         uint16_t queue_id = *(uint16_t *) ret_param;
1722         struct rte_bbdev_info info;
1723         double tb_len_bits;
1724         struct thread_params *tp = cb_arg;
1725
1726         /* Find matching thread params using queue_id */
1727         for (i = 0; i < MAX_QUEUES; ++i, ++tp)
1728                 if (tp->queue_id == queue_id)
1729                         break;
1730
1731         if (i == MAX_QUEUES) {
1732                 printf("%s: Queue_id from interrupt details was not found!\n",
1733                                 __func__);
1734                 return;
1735         }
1736
1737         if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
1738                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1739                 printf(
1740                         "Dequeue interrupt handler called for incorrect event!\n");
1741                 return;
1742         }
1743
1744         burst_sz = rte_atomic16_read(&tp->burst_sz);
1745         num_ops = tp->op_params->num_to_process;
1746
1747         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
1748                         test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
1749                 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1750                                 &tp->dec_ops[
1751                                         rte_atomic16_read(&tp->nb_dequeued)],
1752                                 burst_sz);
1753         else
1754                 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1755                                 &tp->enc_ops[
1756                                         rte_atomic16_read(&tp->nb_dequeued)],
1757                                 burst_sz);
1758
1759         if (deq < burst_sz) {
1760                 printf(
1761                         "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
1762                         burst_sz, deq);
1763                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1764                 return;
1765         }
1766
1767         if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
1768                 rte_atomic16_add(&tp->nb_dequeued, deq);
1769                 return;
1770         }
1771
1772         total_time = rte_rdtsc_precise() - tp->start_time;
1773
1774         rte_bbdev_info_get(dev_id, &info);
1775
1776         ret = TEST_SUCCESS;
1777
1778         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
1779                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1780                 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
1781                                 tp->op_params->vector_mask);
1782                 /* get the max of iter_count for all dequeued ops */
1783                 for (i = 0; i < num_ops; ++i)
1784                         tp->iter_count = RTE_MAX(
1785                                         tp->dec_ops[i]->turbo_dec.iter_count,
1786                                         tp->iter_count);
1787                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
1788         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
1789                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1790                 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
1791                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
1792         } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
1793                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1794                 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
1795                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
1796         } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1797                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1798                 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
1799                                 tp->op_params->vector_mask);
1800                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
1801         }
1802
1803         if (ret) {
1804                 printf("Buffers validation failed\n");
1805                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1806         }
1807
1808         switch (test_vector.op_type) {
1809         case RTE_BBDEV_OP_TURBO_DEC:
1810                 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
1811                 break;
1812         case RTE_BBDEV_OP_TURBO_ENC:
1813                 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
1814                 break;
1815         case RTE_BBDEV_OP_LDPC_DEC:
1816                 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
1817                 break;
1818         case RTE_BBDEV_OP_LDPC_ENC:
1819                 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
1820                 break;
1821         case RTE_BBDEV_OP_NONE:
1822                 tb_len_bits = 0.0;
1823                 break;
1824         default:
1825                 printf("Unknown op type: %d\n", test_vector.op_type);
1826                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1827                 return;
1828         }
1829
1830         tp->ops_per_sec += ((double)num_ops) /
1831                         ((double)total_time / (double)rte_get_tsc_hz());
1832         tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
1833                         ((double)total_time / (double)rte_get_tsc_hz());
1834
1835         rte_atomic16_add(&tp->nb_dequeued, deq);
1836 }
1837
1838 static int
1839 throughput_intr_lcore_dec(void *arg)
1840 {
1841         struct thread_params *tp = arg;
1842         unsigned int enqueued;
1843         const uint16_t queue_id = tp->queue_id;
1844         const uint16_t burst_sz = tp->op_params->burst_sz;
1845         const uint16_t num_to_process = tp->op_params->num_to_process;
1846         struct rte_bbdev_dec_op *ops[num_to_process];
1847         struct test_buffers *bufs = NULL;
1848         struct rte_bbdev_info info;
1849         int ret, i, j;
1850         uint16_t num_to_enq, enq;
1851
1852         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1853                         "BURST_SIZE should be <= %u", MAX_BURST);
1854
1855         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1856                         "Failed to enable interrupts for dev: %u, queue_id: %u",
1857                         tp->dev_id, queue_id);
1858
1859         rte_bbdev_info_get(tp->dev_id, &info);
1860
1861         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
1862                         "NUM_OPS cannot exceed %u for this device",
1863                         info.drv.queue_size_lim);
1864
1865         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1866
1867         rte_atomic16_clear(&tp->processing_status);
1868         rte_atomic16_clear(&tp->nb_dequeued);
1869
1870         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1871                 rte_pause();
1872
1873         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
1874                                 num_to_process);
1875         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
1876                         num_to_process);
1877         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1878                 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
1879                                 bufs->hard_outputs, bufs->soft_outputs,
1880                                 tp->op_params->ref_dec_op);
1881
1882         /* Set counter to validate the ordering */
1883         for (j = 0; j < num_to_process; ++j)
1884                 ops[j]->opaque_data = (void *)(uintptr_t)j;
1885
1886         for (j = 0; j < TEST_REPETITIONS; ++j) {
1887                 for (i = 0; i < num_to_process; ++i)
1888                         rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
1889
1890                 tp->start_time = rte_rdtsc_precise();
1891                 for (enqueued = 0; enqueued < num_to_process;) {
1892                         num_to_enq = burst_sz;
1893
1894                         if (unlikely(num_to_process - enqueued < num_to_enq))
1895                                 num_to_enq = num_to_process - enqueued;
1896
1897                         enq = 0;
1898                         do {
1899                                 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
1900                                                 queue_id, &ops[enqueued],
1901                                                 num_to_enq);
1902                         } while (unlikely(num_to_enq != enq));
1903                         enqueued += enq;
1904
1905                         /* Write to thread burst_sz current number of enqueued
1906                          * descriptors. It ensures that proper number of
1907                          * descriptors will be dequeued in callback
1908                          * function - needed for last batch in case where
1909                          * the number of operations is not a multiple of
1910                          * burst size.
1911                          */
1912                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
1913
1914                         /* Wait until processing of previous batch is
1915                          * completed
1916                          */
1917                         while (rte_atomic16_read(&tp->nb_dequeued) !=
1918                                         (int16_t) enqueued)
1919                                 rte_pause();
1920                 }
1921                 if (j != TEST_REPETITIONS - 1)
1922                         rte_atomic16_clear(&tp->nb_dequeued);
1923         }
1924
1925         return TEST_SUCCESS;
1926 }
1927
1928 static int
1929 throughput_intr_lcore_enc(void *arg)
1930 {
1931         struct thread_params *tp = arg;
1932         unsigned int enqueued;
1933         const uint16_t queue_id = tp->queue_id;
1934         const uint16_t burst_sz = tp->op_params->burst_sz;
1935         const uint16_t num_to_process = tp->op_params->num_to_process;
1936         struct rte_bbdev_enc_op *ops[num_to_process];
1937         struct test_buffers *bufs = NULL;
1938         struct rte_bbdev_info info;
1939         int ret, i, j;
1940         uint16_t num_to_enq, enq;
1941
1942         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1943                         "BURST_SIZE should be <= %u", MAX_BURST);
1944
1945         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1946                         "Failed to enable interrupts for dev: %u, queue_id: %u",
1947                         tp->dev_id, queue_id);
1948
1949         rte_bbdev_info_get(tp->dev_id, &info);
1950
1951         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
1952                         "NUM_OPS cannot exceed %u for this device",
1953                         info.drv.queue_size_lim);
1954
1955         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1956
1957         rte_atomic16_clear(&tp->processing_status);
1958         rte_atomic16_clear(&tp->nb_dequeued);
1959
1960         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1961                 rte_pause();
1962
1963         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
1964                         num_to_process);
1965         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
1966                         num_to_process);
1967         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1968                 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
1969                                 bufs->hard_outputs, tp->op_params->ref_enc_op);
1970
1971         /* Set counter to validate the ordering */
1972         for (j = 0; j < num_to_process; ++j)
1973                 ops[j]->opaque_data = (void *)(uintptr_t)j;
1974
1975         for (j = 0; j < TEST_REPETITIONS; ++j) {
1976                 for (i = 0; i < num_to_process; ++i)
1977                         rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
1978
1979                 tp->start_time = rte_rdtsc_precise();
1980                 for (enqueued = 0; enqueued < num_to_process;) {
1981                         num_to_enq = burst_sz;
1982
1983                         if (unlikely(num_to_process - enqueued < num_to_enq))
1984                                 num_to_enq = num_to_process - enqueued;
1985
1986                         enq = 0;
1987                         do {
1988                                 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
1989                                                 queue_id, &ops[enqueued],
1990                                                 num_to_enq);
1991                         } while (unlikely(enq != num_to_enq));
1992                         enqueued += enq;
1993
1994                         /* Write to thread burst_sz current number of enqueued
1995                          * descriptors. It ensures that proper number of
1996                          * descriptors will be dequeued in callback
1997                          * function - needed for last batch in case where
1998                          * the number of operations is not a multiple of
1999                          * burst size.
2000                          */
2001                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2002
2003                         /* Wait until processing of previous batch is
2004                          * completed
2005                          */
2006                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2007                                         (int16_t) enqueued)
2008                                 rte_pause();
2009                 }
2010                 if (j != TEST_REPETITIONS - 1)
2011                         rte_atomic16_clear(&tp->nb_dequeued);
2012         }
2013
2014         return TEST_SUCCESS;
2015 }
2016
2017 static int
2018 throughput_pmd_lcore_dec(void *arg)
2019 {
2020         struct thread_params *tp = arg;
2021         uint16_t enq, deq;
2022         uint64_t total_time = 0, start_time;
2023         const uint16_t queue_id = tp->queue_id;
2024         const uint16_t burst_sz = tp->op_params->burst_sz;
2025         const uint16_t num_ops = tp->op_params->num_to_process;
2026         struct rte_bbdev_dec_op *ops_enq[num_ops];
2027         struct rte_bbdev_dec_op *ops_deq[num_ops];
2028         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2029         struct test_buffers *bufs = NULL;
2030         int i, j, ret;
2031         struct rte_bbdev_info info;
2032         uint16_t num_to_enq;
2033
2034         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2035                         "BURST_SIZE should be <= %u", MAX_BURST);
2036
2037         rte_bbdev_info_get(tp->dev_id, &info);
2038
2039         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2040                         "NUM_OPS cannot exceed %u for this device",
2041                         info.drv.queue_size_lim);
2042
2043         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2044
2045         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2046                 rte_pause();
2047
2048         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
2049         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
2050
2051         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2052                 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
2053                                 bufs->hard_outputs, bufs->soft_outputs, ref_op);
2054
2055         /* Set counter to validate the ordering */
2056         for (j = 0; j < num_ops; ++j)
2057                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2058
2059         for (i = 0; i < TEST_REPETITIONS; ++i) {
2060
2061                 for (j = 0; j < num_ops; ++j)
2062                         mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
2063
2064                 start_time = rte_rdtsc_precise();
2065
2066                 for (enq = 0, deq = 0; enq < num_ops;) {
2067                         num_to_enq = burst_sz;
2068
2069                         if (unlikely(num_ops - enq < num_to_enq))
2070                                 num_to_enq = num_ops - enq;
2071
2072                         enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2073                                         queue_id, &ops_enq[enq], num_to_enq);
2074
2075                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
2076                                         queue_id, &ops_deq[deq], enq - deq);
2077                 }
2078
2079                 /* dequeue the remaining */
2080                 while (deq < enq) {
2081                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
2082                                         queue_id, &ops_deq[deq], enq - deq);
2083                 }
2084
2085                 total_time += rte_rdtsc_precise() - start_time;
2086         }
2087
2088         tp->iter_count = 0;
2089         /* get the max of iter_count for all dequeued ops */
2090         for (i = 0; i < num_ops; ++i) {
2091                 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
2092                                 tp->iter_count);
2093         }
2094
2095         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2096                 ret = validate_dec_op(ops_deq, num_ops, ref_op,
2097                                 tp->op_params->vector_mask);
2098                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2099         }
2100
2101         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
2102
2103         double tb_len_bits = calc_dec_TB_size(ref_op);
2104
2105         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2106                         ((double)total_time / (double)rte_get_tsc_hz());
2107         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
2108                         1000000.0) / ((double)total_time /
2109                         (double)rte_get_tsc_hz());
2110
2111         return TEST_SUCCESS;
2112 }
2113
2114 static int
2115 throughput_pmd_lcore_ldpc_dec(void *arg)
2116 {
2117         struct thread_params *tp = arg;
2118         uint16_t enq, deq;
2119         uint64_t total_time = 0, start_time;
2120         const uint16_t queue_id = tp->queue_id;
2121         const uint16_t burst_sz = tp->op_params->burst_sz;
2122         const uint16_t num_ops = tp->op_params->num_to_process;
2123         struct rte_bbdev_dec_op *ops_enq[num_ops];
2124         struct rte_bbdev_dec_op *ops_deq[num_ops];
2125         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2126         struct test_buffers *bufs = NULL;
2127         int i, j, ret;
2128         struct rte_bbdev_info info;
2129         uint16_t num_to_enq;
2130
2131         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2132                         "BURST_SIZE should be <= %u", MAX_BURST);
2133
2134         rte_bbdev_info_get(tp->dev_id, &info);
2135
2136         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2137                         "NUM_OPS cannot exceed %u for this device",
2138                         info.drv.queue_size_lim);
2139
2140         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2141
2142         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2143                 rte_pause();
2144
2145         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
2146         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
2147
2148         /* For throughput tests we need to disable early termination */
2149         if (check_bit(ref_op->ldpc_dec.op_flags,
2150                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
2151                 ref_op->ldpc_dec.op_flags -=
2152                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
2153         ref_op->ldpc_dec.iter_max = 6;
2154         ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
2155
2156         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2157                 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
2158                                 bufs->hard_outputs, bufs->soft_outputs,
2159                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2160
2161         /* Set counter to validate the ordering */
2162         for (j = 0; j < num_ops; ++j)
2163                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2164
2165         for (i = 0; i < TEST_REPETITIONS; ++i) {
2166                 for (j = 0; j < num_ops; ++j) {
2167                         mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data);
2168                         if (check_bit(ref_op->ldpc_dec.op_flags,
2169                                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE))
2170                                 mbuf_reset(
2171                                 ops_enq[j]->ldpc_dec.harq_combined_output.data);
2172                 }
2173
2174                 start_time = rte_rdtsc_precise();
2175
2176                 for (enq = 0, deq = 0; enq < num_ops;) {
2177                         num_to_enq = burst_sz;
2178
2179                         if (unlikely(num_ops - enq < num_to_enq))
2180                                 num_to_enq = num_ops - enq;
2181
2182                         enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
2183                                         queue_id, &ops_enq[enq], num_to_enq);
2184
2185                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
2186                                         queue_id, &ops_deq[deq], enq - deq);
2187                 }
2188
2189                 /* dequeue the remaining */
2190                 while (deq < enq) {
2191                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
2192                                         queue_id, &ops_deq[deq], enq - deq);
2193                 }
2194
2195                 total_time += rte_rdtsc_precise() - start_time;
2196         }
2197
2198         tp->iter_count = 0;
2199         /* get the max of iter_count for all dequeued ops */
2200         for (i = 0; i < num_ops; ++i) {
2201                 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
2202                                 tp->iter_count);
2203         }
2204
2205         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2206                 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
2207                                 tp->op_params->vector_mask);
2208                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2209         }
2210
2211         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
2212
2213         double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
2214
2215         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2216                         ((double)total_time / (double)rte_get_tsc_hz());
2217         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
2218                         1000000.0) / ((double)total_time /
2219                         (double)rte_get_tsc_hz());
2220
2221         return TEST_SUCCESS;
2222 }
2223
2224 static int
2225 throughput_pmd_lcore_enc(void *arg)
2226 {
2227         struct thread_params *tp = arg;
2228         uint16_t enq, deq;
2229         uint64_t total_time = 0, start_time;
2230         const uint16_t queue_id = tp->queue_id;
2231         const uint16_t burst_sz = tp->op_params->burst_sz;
2232         const uint16_t num_ops = tp->op_params->num_to_process;
2233         struct rte_bbdev_enc_op *ops_enq[num_ops];
2234         struct rte_bbdev_enc_op *ops_deq[num_ops];
2235         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2236         struct test_buffers *bufs = NULL;
2237         int i, j, ret;
2238         struct rte_bbdev_info info;
2239         uint16_t num_to_enq;
2240
2241         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2242                         "BURST_SIZE should be <= %u", MAX_BURST);
2243
2244         rte_bbdev_info_get(tp->dev_id, &info);
2245
2246         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2247                         "NUM_OPS cannot exceed %u for this device",
2248                         info.drv.queue_size_lim);
2249
2250         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2251
2252         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2253                 rte_pause();
2254
2255         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
2256                         num_ops);
2257         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2258                         num_ops);
2259         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2260                 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
2261                                 bufs->hard_outputs, ref_op);
2262
2263         /* Set counter to validate the ordering */
2264         for (j = 0; j < num_ops; ++j)
2265                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2266
2267         for (i = 0; i < TEST_REPETITIONS; ++i) {
2268
2269                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2270                         for (j = 0; j < num_ops; ++j)
2271                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
2272
2273                 start_time = rte_rdtsc_precise();
2274
2275                 for (enq = 0, deq = 0; enq < num_ops;) {
2276                         num_to_enq = burst_sz;
2277
2278                         if (unlikely(num_ops - enq < num_to_enq))
2279                                 num_to_enq = num_ops - enq;
2280
2281                         enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2282                                         queue_id, &ops_enq[enq], num_to_enq);
2283
2284                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
2285                                         queue_id, &ops_deq[deq], enq - deq);
2286                 }
2287
2288                 /* dequeue the remaining */
2289                 while (deq < enq) {
2290                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
2291                                         queue_id, &ops_deq[deq], enq - deq);
2292                 }
2293
2294                 total_time += rte_rdtsc_precise() - start_time;
2295         }
2296
2297         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2298                 ret = validate_enc_op(ops_deq, num_ops, ref_op);
2299                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2300         }
2301
2302         rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
2303
2304         double tb_len_bits = calc_enc_TB_size(ref_op);
2305
2306         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2307                         ((double)total_time / (double)rte_get_tsc_hz());
2308         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
2309                         / 1000000.0) / ((double)total_time /
2310                         (double)rte_get_tsc_hz());
2311
2312         return TEST_SUCCESS;
2313 }
2314
2315 static int
2316 throughput_pmd_lcore_ldpc_enc(void *arg)
2317 {
2318         struct thread_params *tp = arg;
2319         uint16_t enq, deq;
2320         uint64_t total_time = 0, start_time;
2321         const uint16_t queue_id = tp->queue_id;
2322         const uint16_t burst_sz = tp->op_params->burst_sz;
2323         const uint16_t num_ops = tp->op_params->num_to_process;
2324         struct rte_bbdev_enc_op *ops_enq[num_ops];
2325         struct rte_bbdev_enc_op *ops_deq[num_ops];
2326         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2327         struct test_buffers *bufs = NULL;
2328         int i, j, ret;
2329         struct rte_bbdev_info info;
2330         uint16_t num_to_enq;
2331
2332         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2333                         "BURST_SIZE should be <= %u", MAX_BURST);
2334
2335         rte_bbdev_info_get(tp->dev_id, &info);
2336
2337         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2338                         "NUM_OPS cannot exceed %u for this device",
2339                         info.drv.queue_size_lim);
2340
2341         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2342
2343         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2344                 rte_pause();
2345
2346         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
2347                         num_ops);
2348         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2349                         num_ops);
2350         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2351                 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
2352                                 bufs->hard_outputs, ref_op);
2353
2354         /* Set counter to validate the ordering */
2355         for (j = 0; j < num_ops; ++j)
2356                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2357
2358         for (i = 0; i < TEST_REPETITIONS; ++i) {
2359
2360                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2361                         for (j = 0; j < num_ops; ++j)
2362                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
2363
2364                 start_time = rte_rdtsc_precise();
2365
2366                 for (enq = 0, deq = 0; enq < num_ops;) {
2367                         num_to_enq = burst_sz;
2368
2369                         if (unlikely(num_ops - enq < num_to_enq))
2370                                 num_to_enq = num_ops - enq;
2371
2372                         enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
2373                                         queue_id, &ops_enq[enq], num_to_enq);
2374
2375                         deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
2376                                         queue_id, &ops_deq[deq], enq - deq);
2377                 }
2378
2379                 /* dequeue the remaining */
2380                 while (deq < enq) {
2381                         deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
2382                                         queue_id, &ops_deq[deq], enq - deq);
2383                 }
2384
2385                 total_time += rte_rdtsc_precise() - start_time;
2386         }
2387
2388         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2389                 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
2390                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2391         }
2392
2393         rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
2394
2395         double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
2396
2397         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2398                         ((double)total_time / (double)rte_get_tsc_hz());
2399         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
2400                         / 1000000.0) / ((double)total_time /
2401                         (double)rte_get_tsc_hz());
2402
2403         return TEST_SUCCESS;
2404 }
2405
2406 static void
2407 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
2408 {
2409         unsigned int iter = 0;
2410         double total_mops = 0, total_mbps = 0;
2411
2412         for (iter = 0; iter < used_cores; iter++) {
2413                 printf(
2414                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
2415                         t_params[iter].lcore_id, t_params[iter].ops_per_sec,
2416                         t_params[iter].mbps);
2417                 total_mops += t_params[iter].ops_per_sec;
2418                 total_mbps += t_params[iter].mbps;
2419         }
2420         printf(
2421                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
2422                 used_cores, total_mops, total_mbps);
2423 }
2424
2425 static void
2426 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
2427 {
2428         unsigned int iter = 0;
2429         double total_mops = 0, total_mbps = 0;
2430         uint8_t iter_count = 0;
2431
2432         for (iter = 0; iter < used_cores; iter++) {
2433                 printf(
2434                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
2435                         t_params[iter].lcore_id, t_params[iter].ops_per_sec,
2436                         t_params[iter].mbps, t_params[iter].iter_count);
2437                 total_mops += t_params[iter].ops_per_sec;
2438                 total_mbps += t_params[iter].mbps;
2439                 iter_count = RTE_MAX(iter_count, t_params[iter].iter_count);
2440         }
2441         printf(
2442                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
2443                 used_cores, total_mops, total_mbps, iter_count);
2444 }
2445
2446 /*
2447  * Test function that determines how long an enqueue + dequeue of a burst
2448  * takes on available lcores.
2449  */
2450 static int
2451 throughput_test(struct active_device *ad,
2452                 struct test_op_params *op_params)
2453 {
2454         int ret;
2455         unsigned int lcore_id, used_cores = 0;
2456         struct thread_params *t_params, *tp;
2457         struct rte_bbdev_info info;
2458         lcore_function_t *throughput_function;
2459         uint16_t num_lcores;
2460         const char *op_type_str;
2461
2462         rte_bbdev_info_get(ad->dev_id, &info);
2463
2464         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
2465         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
2466                         test_vector.op_type);
2467
2468         printf("+ ------------------------------------------------------- +\n");
2469         printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
2470                         info.dev_name, ad->nb_queues, op_params->burst_sz,
2471                         op_params->num_to_process, op_params->num_lcores,
2472                         op_type_str,
2473                         intr_enabled ? "Interrupt mode" : "PMD mode",
2474                         (double)rte_get_tsc_hz() / 1000000000.0);
2475
2476         /* Set number of lcores */
2477         num_lcores = (ad->nb_queues < (op_params->num_lcores))
2478                         ? ad->nb_queues
2479                         : op_params->num_lcores;
2480
2481         /* Allocate memory for thread parameters structure */
2482         t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
2483                         RTE_CACHE_LINE_SIZE);
2484         TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
2485                         RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
2486                                 RTE_CACHE_LINE_SIZE));
2487
2488         if (intr_enabled) {
2489                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2490                         throughput_function = throughput_intr_lcore_dec;
2491                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2492                         throughput_function = throughput_intr_lcore_dec;
2493                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2494                         throughput_function = throughput_intr_lcore_enc;
2495                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2496                         throughput_function = throughput_intr_lcore_enc;
2497                 else
2498                         throughput_function = throughput_intr_lcore_enc;
2499
2500                 /* Dequeue interrupt callback registration */
2501                 ret = rte_bbdev_callback_register(ad->dev_id,
2502                                 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
2503                                 t_params);
2504                 if (ret < 0) {
2505                         rte_free(t_params);
2506                         return ret;
2507                 }
2508         } else {
2509                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2510                         throughput_function = throughput_pmd_lcore_dec;
2511                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2512                         throughput_function = throughput_pmd_lcore_ldpc_dec;
2513                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2514                         throughput_function = throughput_pmd_lcore_enc;
2515                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2516                         throughput_function = throughput_pmd_lcore_ldpc_enc;
2517                 else
2518                         throughput_function = throughput_pmd_lcore_enc;
2519         }
2520
2521         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
2522
2523         /* Master core is set at first entry */
2524         t_params[0].dev_id = ad->dev_id;
2525         t_params[0].lcore_id = rte_lcore_id();
2526         t_params[0].op_params = op_params;
2527         t_params[0].queue_id = ad->queue_ids[used_cores++];
2528         t_params[0].iter_count = 0;
2529
2530         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
2531                 if (used_cores >= num_lcores)
2532                         break;
2533
2534                 t_params[used_cores].dev_id = ad->dev_id;
2535                 t_params[used_cores].lcore_id = lcore_id;
2536                 t_params[used_cores].op_params = op_params;
2537                 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
2538                 t_params[used_cores].iter_count = 0;
2539
2540                 rte_eal_remote_launch(throughput_function,
2541                                 &t_params[used_cores++], lcore_id);
2542         }
2543
2544         rte_atomic16_set(&op_params->sync, SYNC_START);
2545         ret = throughput_function(&t_params[0]);
2546
2547         /* Master core is always used */
2548         for (used_cores = 1; used_cores < num_lcores; used_cores++)
2549                 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
2550
2551         /* Return if test failed */
2552         if (ret) {
2553                 rte_free(t_params);
2554                 return ret;
2555         }
2556
2557         /* Print throughput if interrupts are disabled and test passed */
2558         if (!intr_enabled) {
2559                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
2560                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2561                         print_dec_throughput(t_params, num_lcores);
2562                 else
2563                         print_enc_throughput(t_params, num_lcores);
2564                 rte_free(t_params);
2565                 return ret;
2566         }
2567
2568         /* In interrupt TC we need to wait for the interrupt callback to deqeue
2569          * all pending operations. Skip waiting for queues which reported an
2570          * error using processing_status variable.
2571          * Wait for master lcore operations.
2572          */
2573         tp = &t_params[0];
2574         while ((rte_atomic16_read(&tp->nb_dequeued) <
2575                         op_params->num_to_process) &&
2576                         (rte_atomic16_read(&tp->processing_status) !=
2577                         TEST_FAILED))
2578                 rte_pause();
2579
2580         tp->ops_per_sec /= TEST_REPETITIONS;
2581         tp->mbps /= TEST_REPETITIONS;
2582         ret |= (int)rte_atomic16_read(&tp->processing_status);
2583
2584         /* Wait for slave lcores operations */
2585         for (used_cores = 1; used_cores < num_lcores; used_cores++) {
2586                 tp = &t_params[used_cores];
2587
2588                 while ((rte_atomic16_read(&tp->nb_dequeued) <
2589                                 op_params->num_to_process) &&
2590                                 (rte_atomic16_read(&tp->processing_status) !=
2591                                 TEST_FAILED))
2592                         rte_pause();
2593
2594                 tp->ops_per_sec /= TEST_REPETITIONS;
2595                 tp->mbps /= TEST_REPETITIONS;
2596                 ret |= (int)rte_atomic16_read(&tp->processing_status);
2597         }
2598
2599         /* Print throughput if test passed */
2600         if (!ret) {
2601                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
2602                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2603                         print_dec_throughput(t_params, num_lcores);
2604                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
2605                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2606                         print_enc_throughput(t_params, num_lcores);
2607         }
2608
2609         rte_free(t_params);
2610         return ret;
2611 }
2612
2613 static int
2614 latency_test_dec(struct rte_mempool *mempool,
2615                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
2616                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
2617                 const uint16_t num_to_process, uint16_t burst_sz,
2618                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
2619 {
2620         int ret = TEST_SUCCESS;
2621         uint16_t i, j, dequeued;
2622         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2623         uint64_t start_time = 0, last_time = 0;
2624
2625         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2626                 uint16_t enq = 0, deq = 0;
2627                 bool first_time = true;
2628                 last_time = 0;
2629
2630                 if (unlikely(num_to_process - dequeued < burst_sz))
2631                         burst_sz = num_to_process - dequeued;
2632
2633                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
2634                 TEST_ASSERT_SUCCESS(ret,
2635                                 "rte_bbdev_dec_op_alloc_bulk() failed");
2636                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2637                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
2638                                         bufs->inputs,
2639                                         bufs->hard_outputs,
2640                                         bufs->soft_outputs,
2641                                         ref_op);
2642
2643                 /* Set counter to validate the ordering */
2644                 for (j = 0; j < burst_sz; ++j)
2645                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2646
2647                 start_time = rte_rdtsc_precise();
2648
2649                 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
2650                                 burst_sz);
2651                 TEST_ASSERT(enq == burst_sz,
2652                                 "Error enqueueing burst, expected %u, got %u",
2653                                 burst_sz, enq);
2654
2655                 /* Dequeue */
2656                 do {
2657                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2658                                         &ops_deq[deq], burst_sz - deq);
2659                         if (likely(first_time && (deq > 0))) {
2660                                 last_time = rte_rdtsc_precise() - start_time;
2661                                 first_time = false;
2662                         }
2663                 } while (unlikely(burst_sz != deq));
2664
2665                 *max_time = RTE_MAX(*max_time, last_time);
2666                 *min_time = RTE_MIN(*min_time, last_time);
2667                 *total_time += last_time;
2668
2669                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2670                         ret = validate_dec_op(ops_deq, burst_sz, ref_op,
2671                                         vector_mask);
2672                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2673                 }
2674
2675                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
2676                 dequeued += deq;
2677         }
2678
2679         return i;
2680 }
2681
2682 static int
2683 latency_test_ldpc_dec(struct rte_mempool *mempool,
2684                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
2685                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
2686                 const uint16_t num_to_process, uint16_t burst_sz,
2687                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
2688 {
2689         int ret = TEST_SUCCESS;
2690         uint16_t i, j, dequeued;
2691         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2692         uint64_t start_time = 0, last_time = 0;
2693
2694         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2695                 uint16_t enq = 0, deq = 0;
2696                 bool first_time = true;
2697                 last_time = 0;
2698
2699                 if (unlikely(num_to_process - dequeued < burst_sz))
2700                         burst_sz = num_to_process - dequeued;
2701
2702                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
2703                 TEST_ASSERT_SUCCESS(ret,
2704                                 "rte_bbdev_dec_op_alloc_bulk() failed");
2705                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2706                         copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
2707                                         bufs->inputs,
2708                                         bufs->hard_outputs,
2709                                         bufs->soft_outputs,
2710                                         bufs->harq_inputs,
2711                                         bufs->harq_outputs,
2712                                         ref_op);
2713
2714                 /* Set counter to validate the ordering */
2715                 for (j = 0; j < burst_sz; ++j)
2716                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2717
2718                 start_time = rte_rdtsc_precise();
2719
2720                 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2721                                 &ops_enq[enq], burst_sz);
2722                 TEST_ASSERT(enq == burst_sz,
2723                                 "Error enqueueing burst, expected %u, got %u",
2724                                 burst_sz, enq);
2725
2726                 /* Dequeue */
2727                 do {
2728                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2729                                         &ops_deq[deq], burst_sz - deq);
2730                         if (likely(first_time && (deq > 0))) {
2731                                 last_time = rte_rdtsc_precise() - start_time;
2732                                 first_time = false;
2733                         }
2734                 } while (unlikely(burst_sz != deq));
2735
2736                 *max_time = RTE_MAX(*max_time, last_time);
2737                 *min_time = RTE_MIN(*min_time, last_time);
2738                 *total_time += last_time;
2739
2740                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2741                         ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
2742                                         vector_mask);
2743                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2744                 }
2745
2746                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
2747                 dequeued += deq;
2748         }
2749
2750         return i;
2751 }
2752
2753 static int
2754 latency_test_enc(struct rte_mempool *mempool,
2755                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
2756                 uint16_t dev_id, uint16_t queue_id,
2757                 const uint16_t num_to_process, uint16_t burst_sz,
2758                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
2759 {
2760         int ret = TEST_SUCCESS;
2761         uint16_t i, j, dequeued;
2762         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2763         uint64_t start_time = 0, last_time = 0;
2764
2765         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2766                 uint16_t enq = 0, deq = 0;
2767                 bool first_time = true;
2768                 last_time = 0;
2769
2770                 if (unlikely(num_to_process - dequeued < burst_sz))
2771                         burst_sz = num_to_process - dequeued;
2772
2773                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
2774                 TEST_ASSERT_SUCCESS(ret,
2775                                 "rte_bbdev_enc_op_alloc_bulk() failed");
2776                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2777                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
2778                                         bufs->inputs,
2779                                         bufs->hard_outputs,
2780                                         ref_op);
2781
2782                 /* Set counter to validate the ordering */
2783                 for (j = 0; j < burst_sz; ++j)
2784                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2785
2786                 start_time = rte_rdtsc_precise();
2787
2788                 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
2789                                 burst_sz);
2790                 TEST_ASSERT(enq == burst_sz,
2791                                 "Error enqueueing burst, expected %u, got %u",
2792                                 burst_sz, enq);
2793
2794                 /* Dequeue */
2795                 do {
2796                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2797                                         &ops_deq[deq], burst_sz - deq);
2798                         if (likely(first_time && (deq > 0))) {
2799                                 last_time += rte_rdtsc_precise() - start_time;
2800                                 first_time = false;
2801                         }
2802                 } while (unlikely(burst_sz != deq));
2803
2804                 *max_time = RTE_MAX(*max_time, last_time);
2805                 *min_time = RTE_MIN(*min_time, last_time);
2806                 *total_time += last_time;
2807
2808                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2809                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
2810                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2811                 }
2812
2813                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
2814                 dequeued += deq;
2815         }
2816
2817         return i;
2818 }
2819
2820 static int
2821 latency_test_ldpc_enc(struct rte_mempool *mempool,
2822                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
2823                 uint16_t dev_id, uint16_t queue_id,
2824                 const uint16_t num_to_process, uint16_t burst_sz,
2825                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
2826 {
2827         int ret = TEST_SUCCESS;
2828         uint16_t i, j, dequeued;
2829         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2830         uint64_t start_time = 0, last_time = 0;
2831
2832         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2833                 uint16_t enq = 0, deq = 0;
2834                 bool first_time = true;
2835                 last_time = 0;
2836
2837                 if (unlikely(num_to_process - dequeued < burst_sz))
2838                         burst_sz = num_to_process - dequeued;
2839
2840                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
2841
2842                 TEST_ASSERT_SUCCESS(ret,
2843                                 "rte_bbdev_enc_op_alloc_bulk() failed");
2844                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2845                         copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
2846                                         bufs->inputs,
2847                                         bufs->hard_outputs,
2848                                         ref_op);
2849
2850                 /* Set counter to validate the ordering */
2851                 for (j = 0; j < burst_sz; ++j)
2852                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2853
2854                 start_time = rte_rdtsc_precise();
2855
2856                 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
2857                                 &ops_enq[enq], burst_sz);
2858                 TEST_ASSERT(enq == burst_sz,
2859                                 "Error enqueueing burst, expected %u, got %u",
2860                                 burst_sz, enq);
2861
2862                 /* Dequeue */
2863                 do {
2864                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2865                                         &ops_deq[deq], burst_sz - deq);
2866                         if (likely(first_time && (deq > 0))) {
2867                                 last_time += rte_rdtsc_precise() - start_time;
2868                                 first_time = false;
2869                         }
2870                 } while (unlikely(burst_sz != deq));
2871
2872                 *max_time = RTE_MAX(*max_time, last_time);
2873                 *min_time = RTE_MIN(*min_time, last_time);
2874                 *total_time += last_time;
2875
2876                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2877                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
2878                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2879                 }
2880
2881                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
2882                 dequeued += deq;
2883         }
2884
2885         return i;
2886 }
2887
2888 static int
2889 latency_test(struct active_device *ad,
2890                 struct test_op_params *op_params)
2891 {
2892         int iter;
2893         uint16_t burst_sz = op_params->burst_sz;
2894         const uint16_t num_to_process = op_params->num_to_process;
2895         const enum rte_bbdev_op_type op_type = test_vector.op_type;
2896         const uint16_t queue_id = ad->queue_ids[0];
2897         struct test_buffers *bufs = NULL;
2898         struct rte_bbdev_info info;
2899         uint64_t total_time, min_time, max_time;
2900         const char *op_type_str;
2901
2902         total_time = max_time = 0;
2903         min_time = UINT64_MAX;
2904
2905         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2906                         "BURST_SIZE should be <= %u", MAX_BURST);
2907
2908         rte_bbdev_info_get(ad->dev_id, &info);
2909         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2910
2911         op_type_str = rte_bbdev_op_type_str(op_type);
2912         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2913
2914         printf("+ ------------------------------------------------------- +\n");
2915         printf("== test: validation/latency\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
2916                         info.dev_name, burst_sz, num_to_process, op_type_str);
2917
2918         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2919                 iter = latency_test_dec(op_params->mp, bufs,
2920                                 op_params->ref_dec_op, op_params->vector_mask,
2921                                 ad->dev_id, queue_id, num_to_process,
2922                                 burst_sz, &total_time, &min_time, &max_time);
2923         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
2924                 iter = latency_test_enc(op_params->mp, bufs,
2925                                 op_params->ref_enc_op, ad->dev_id, queue_id,
2926                                 num_to_process, burst_sz, &total_time,
2927                                 &min_time, &max_time);
2928         else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
2929                 iter = latency_test_ldpc_enc(op_params->mp, bufs,
2930                                 op_params->ref_enc_op, ad->dev_id, queue_id,
2931                                 num_to_process, burst_sz, &total_time,
2932                                 &min_time, &max_time);
2933         else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
2934                 iter = latency_test_ldpc_dec(op_params->mp, bufs,
2935                                 op_params->ref_dec_op, op_params->vector_mask,
2936                                 ad->dev_id, queue_id, num_to_process,
2937                                 burst_sz, &total_time, &min_time, &max_time);
2938         else
2939                 iter = latency_test_enc(op_params->mp, bufs,
2940                                         op_params->ref_enc_op,
2941                                         ad->dev_id, queue_id,
2942                                         num_to_process, burst_sz, &total_time,
2943                                         &min_time, &max_time);
2944
2945         if (iter <= 0)
2946                 return TEST_FAILED;
2947
2948         printf("Operation latency:\n"
2949                         "\tavg: %lg cycles, %lg us\n"
2950                         "\tmin: %lg cycles, %lg us\n"
2951                         "\tmax: %lg cycles, %lg us\n",
2952                         (double)total_time / (double)iter,
2953                         (double)(total_time * 1000000) / (double)iter /
2954                         (double)rte_get_tsc_hz(), (double)min_time,
2955                         (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
2956                         (double)max_time, (double)(max_time * 1000000) /
2957                         (double)rte_get_tsc_hz());
2958
2959         return TEST_SUCCESS;
2960 }
2961
2962 #ifdef RTE_BBDEV_OFFLOAD_COST
2963 static int
2964 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
2965                 struct rte_bbdev_stats *stats)
2966 {
2967         struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
2968         struct rte_bbdev_stats *q_stats;
2969
2970         if (queue_id >= dev->data->num_queues)
2971                 return -1;
2972
2973         q_stats = &dev->data->queues[queue_id].queue_stats;
2974
2975         stats->enqueued_count = q_stats->enqueued_count;
2976         stats->dequeued_count = q_stats->dequeued_count;
2977         stats->enqueue_err_count = q_stats->enqueue_err_count;
2978         stats->dequeue_err_count = q_stats->dequeue_err_count;
2979         stats->acc_offload_cycles = q_stats->acc_offload_cycles;
2980
2981         return 0;
2982 }
2983
2984 static int
2985 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
2986                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
2987                 uint16_t queue_id, const uint16_t num_to_process,
2988                 uint16_t burst_sz, struct test_time_stats *time_st)
2989 {
2990         int i, dequeued, ret;
2991         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2992         uint64_t enq_start_time, deq_start_time;
2993         uint64_t enq_sw_last_time, deq_last_time;
2994         struct rte_bbdev_stats stats;
2995
2996         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2997                 uint16_t enq = 0, deq = 0;
2998
2999                 if (unlikely(num_to_process - dequeued < burst_sz))
3000                         burst_sz = num_to_process - dequeued;
3001
3002                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3003                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3004                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
3005                                         bufs->inputs,
3006                                         bufs->hard_outputs,
3007                                         bufs->soft_outputs,
3008                                         ref_op);
3009
3010                 /* Start time meas for enqueue function offload latency */
3011                 enq_start_time = rte_rdtsc_precise();
3012                 do {
3013                         enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
3014                                         &ops_enq[enq], burst_sz - enq);
3015                 } while (unlikely(burst_sz != enq));
3016
3017                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
3018                 TEST_ASSERT_SUCCESS(ret,
3019                                 "Failed to get stats for queue (%u) of device (%u)",
3020                                 queue_id, dev_id);
3021
3022                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
3023                                 stats.acc_offload_cycles;
3024                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
3025                                 enq_sw_last_time);
3026                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
3027                                 enq_sw_last_time);
3028                 time_st->enq_sw_total_time += enq_sw_last_time;
3029
3030                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
3031                                 stats.acc_offload_cycles);
3032                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
3033                                 stats.acc_offload_cycles);
3034                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
3035
3036                 /* give time for device to process ops */
3037                 rte_delay_us(200);
3038
3039                 /* Start time meas for dequeue function offload latency */
3040                 deq_start_time = rte_rdtsc_precise();
3041                 /* Dequeue one operation */
3042                 do {
3043                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3044                                         &ops_deq[deq], 1);
3045                 } while (unlikely(deq != 1));
3046
3047                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3048                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
3049                                 deq_last_time);
3050                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
3051                                 deq_last_time);
3052                 time_st->deq_total_time += deq_last_time;
3053
3054                 /* Dequeue remaining operations if needed*/
3055                 while (burst_sz != deq)
3056                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3057                                         &ops_deq[deq], burst_sz - deq);
3058
3059                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3060                 dequeued += deq;
3061         }
3062
3063         return i;
3064 }
3065
3066 static int
3067 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
3068                 struct test_buffers *bufs,
3069                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
3070                 uint16_t queue_id, const uint16_t num_to_process,
3071                 uint16_t burst_sz, struct test_time_stats *time_st)
3072 {
3073         int i, dequeued, ret;
3074         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3075         uint64_t enq_start_time, deq_start_time;
3076         uint64_t enq_sw_last_time, deq_last_time;
3077         struct rte_bbdev_stats stats;
3078
3079         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3080                 uint16_t enq = 0, deq = 0;
3081
3082                 if (unlikely(num_to_process - dequeued < burst_sz))
3083                         burst_sz = num_to_process - dequeued;
3084
3085                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3086                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3087                         copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
3088                                         bufs->inputs,
3089                                         bufs->hard_outputs,
3090                                         bufs->soft_outputs,
3091                                         bufs->harq_inputs,
3092                                         bufs->harq_outputs,
3093                                         ref_op);
3094
3095                 /* Start time meas for enqueue function offload latency */
3096                 enq_start_time = rte_rdtsc_precise();
3097                 do {
3098                         enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
3099                                         &ops_enq[enq], burst_sz - enq);
3100                 } while (unlikely(burst_sz != enq));
3101
3102                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
3103                 TEST_ASSERT_SUCCESS(ret,
3104                                 "Failed to get stats for queue (%u) of device (%u)",
3105                                 queue_id, dev_id);
3106
3107                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
3108                                 stats.acc_offload_cycles;
3109                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
3110                                 enq_sw_last_time);
3111                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
3112                                 enq_sw_last_time);
3113                 time_st->enq_sw_total_time += enq_sw_last_time;
3114
3115                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
3116                                 stats.acc_offload_cycles);
3117                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
3118                                 stats.acc_offload_cycles);
3119                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
3120
3121                 /* give time for device to process ops */
3122                 rte_delay_us(200);
3123
3124                 /* Start time meas for dequeue function offload latency */
3125                 deq_start_time = rte_rdtsc_precise();
3126                 /* Dequeue one operation */
3127                 do {
3128                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
3129                                         &ops_deq[deq], 1);
3130                 } while (unlikely(deq != 1));
3131
3132                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3133                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
3134                                 deq_last_time);
3135                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
3136                                 deq_last_time);
3137                 time_st->deq_total_time += deq_last_time;
3138
3139                 /* Dequeue remaining operations if needed*/
3140                 while (burst_sz != deq)
3141                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3142                                         &ops_deq[deq], burst_sz - deq);
3143
3144                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3145                 dequeued += deq;
3146         }
3147
3148         return i;
3149 }
3150
3151 static int
3152 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
3153                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
3154                 uint16_t queue_id, const uint16_t num_to_process,
3155                 uint16_t burst_sz, struct test_time_stats *time_st)
3156 {
3157         int i, dequeued, ret;
3158         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3159         uint64_t enq_start_time, deq_start_time;
3160         uint64_t enq_sw_last_time, deq_last_time;
3161         struct rte_bbdev_stats stats;
3162
3163         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3164                 uint16_t enq = 0, deq = 0;
3165
3166                 if (unlikely(num_to_process - dequeued < burst_sz))
3167                         burst_sz = num_to_process - dequeued;
3168
3169                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
3170                 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
3171                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3172                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
3173                                         bufs->inputs,
3174                                         bufs->hard_outputs,
3175                                         ref_op);
3176
3177                 /* Start time meas for enqueue function offload latency */
3178                 enq_start_time = rte_rdtsc_precise();
3179                 do {
3180                         enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
3181                                         &ops_enq[enq], burst_sz - enq);
3182                 } while (unlikely(burst_sz != enq));
3183
3184                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
3185                 TEST_ASSERT_SUCCESS(ret,
3186                                 "Failed to get stats for queue (%u) of device (%u)",
3187                                 queue_id, dev_id);
3188
3189                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
3190                                 stats.acc_offload_cycles;
3191                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
3192                                 enq_sw_last_time);
3193                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
3194                                 enq_sw_last_time);
3195                 time_st->enq_sw_total_time += enq_sw_last_time;
3196
3197                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
3198                                 stats.acc_offload_cycles);
3199                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
3200                                 stats.acc_offload_cycles);
3201                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
3202
3203                 /* give time for device to process ops */
3204                 rte_delay_us(200);
3205
3206                 /* Start time meas for dequeue function offload latency */
3207                 deq_start_time = rte_rdtsc_precise();
3208                 /* Dequeue one operation */
3209                 do {
3210                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
3211                                         &ops_deq[deq], 1);
3212                 } while (unlikely(deq != 1));
3213
3214                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3215                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
3216                                 deq_last_time);
3217                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
3218                                 deq_last_time);
3219                 time_st->deq_total_time += deq_last_time;
3220
3221                 while (burst_sz != deq)
3222                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
3223                                         &ops_deq[deq], burst_sz - deq);
3224
3225                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
3226                 dequeued += deq;
3227         }
3228
3229         return i;
3230 }
3231
3232 static int
3233 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
3234                 struct test_buffers *bufs,
3235                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
3236                 uint16_t queue_id, const uint16_t num_to_process,
3237                 uint16_t burst_sz, struct test_time_stats *time_st)
3238 {
3239         int i, dequeued, ret;
3240         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3241         uint64_t enq_start_time, deq_start_time;
3242         uint64_t enq_sw_last_time, deq_last_time;
3243         struct rte_bbdev_stats stats;
3244
3245         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3246                 uint16_t enq = 0, deq = 0;
3247
3248                 if (unlikely(num_to_process - dequeued < burst_sz))
3249                         burst_sz = num_to_process - dequeued;
3250
3251                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
3252                 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
3253                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3254                         copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
3255                                         bufs->inputs,
3256                                         bufs->hard_outputs,
3257                                         ref_op);
3258
3259                 /* Start time meas for enqueue function offload latency */
3260                 enq_start_time = rte_rdtsc_precise();
3261                 do {
3262                         enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
3263                                         &ops_enq[enq], burst_sz - enq);
3264                 } while (unlikely(burst_sz != enq));
3265
3266                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
3267                 TEST_ASSERT_SUCCESS(ret,
3268                                 "Failed to get stats for queue (%u) of device (%u)",
3269                                 queue_id, dev_id);
3270
3271                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
3272                                 stats.acc_offload_cycles;
3273                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
3274                                 enq_sw_last_time);
3275                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
3276                                 enq_sw_last_time);
3277                 time_st->enq_sw_total_time += enq_sw_last_time;
3278
3279                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
3280                                 stats.acc_offload_cycles);
3281                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
3282                                 stats.acc_offload_cycles);
3283                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
3284
3285                 /* give time for device to process ops */
3286                 rte_delay_us(200);
3287
3288                 /* Start time meas for dequeue function offload latency */
3289                 deq_start_time = rte_rdtsc_precise();
3290                 /* Dequeue one operation */
3291                 do {
3292                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
3293                                         &ops_deq[deq], 1);
3294                 } while (unlikely(deq != 1));
3295
3296                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3297                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
3298                                 deq_last_time);
3299                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
3300                                 deq_last_time);
3301                 time_st->deq_total_time += deq_last_time;
3302
3303                 while (burst_sz != deq)
3304                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
3305                                         &ops_deq[deq], burst_sz - deq);
3306
3307                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
3308                 dequeued += deq;
3309         }
3310
3311         return i;
3312 }
3313 #endif
3314
3315 static int
3316 offload_cost_test(struct active_device *ad,
3317                 struct test_op_params *op_params)
3318 {
3319 #ifndef RTE_BBDEV_OFFLOAD_COST
3320         RTE_SET_USED(ad);
3321         RTE_SET_USED(op_params);
3322         printf("Offload latency test is disabled.\n");
3323         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
3324         return TEST_SKIPPED;
3325 #else
3326         int iter;
3327         uint16_t burst_sz = op_params->burst_sz;
3328         const uint16_t num_to_process = op_params->num_to_process;
3329         const enum rte_bbdev_op_type op_type = test_vector.op_type;
3330         const uint16_t queue_id = ad->queue_ids[0];
3331         struct test_buffers *bufs = NULL;
3332         struct rte_bbdev_info info;
3333         const char *op_type_str;
3334         struct test_time_stats time_st;
3335
3336         memset(&time_st, 0, sizeof(struct test_time_stats));
3337         time_st.enq_sw_min_time = UINT64_MAX;
3338         time_st.enq_acc_min_time = UINT64_MAX;
3339         time_st.deq_min_time = UINT64_MAX;
3340
3341         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3342                         "BURST_SIZE should be <= %u", MAX_BURST);
3343
3344         rte_bbdev_info_get(ad->dev_id, &info);
3345         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3346
3347         op_type_str = rte_bbdev_op_type_str(op_type);
3348         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
3349
3350         printf("+ ------------------------------------------------------- +\n");
3351         printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
3352                         info.dev_name, burst_sz, num_to_process, op_type_str);
3353
3354         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
3355                 iter = offload_latency_test_dec(op_params->mp, bufs,
3356                                 op_params->ref_dec_op, ad->dev_id, queue_id,
3357                                 num_to_process, burst_sz, &time_st);
3358         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
3359                 iter = offload_latency_test_enc(op_params->mp, bufs,
3360                                 op_params->ref_enc_op, ad->dev_id, queue_id,
3361                                 num_to_process, burst_sz, &time_st);
3362         else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
3363                 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
3364                                 op_params->ref_enc_op, ad->dev_id, queue_id,
3365                                 num_to_process, burst_sz, &time_st);
3366         else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
3367                 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
3368                         op_params->ref_dec_op, ad->dev_id, queue_id,
3369                         num_to_process, burst_sz, &time_st);
3370         else
3371                 iter = offload_latency_test_enc(op_params->mp, bufs,
3372                                 op_params->ref_enc_op, ad->dev_id, queue_id,
3373                                 num_to_process, burst_sz, &time_st);
3374
3375         if (iter <= 0)
3376                 return TEST_FAILED;
3377
3378         printf("Enqueue driver offload cost latency:\n"
3379                         "\tavg: %lg cycles, %lg us\n"
3380                         "\tmin: %lg cycles, %lg us\n"
3381                         "\tmax: %lg cycles, %lg us\n"
3382                         "Enqueue accelerator offload cost latency:\n"
3383                         "\tavg: %lg cycles, %lg us\n"
3384                         "\tmin: %lg cycles, %lg us\n"
3385                         "\tmax: %lg cycles, %lg us\n",
3386                         (double)time_st.enq_sw_total_time / (double)iter,
3387                         (double)(time_st.enq_sw_total_time * 1000000) /
3388                         (double)iter / (double)rte_get_tsc_hz(),
3389                         (double)time_st.enq_sw_min_time,
3390                         (double)(time_st.enq_sw_min_time * 1000000) /
3391                         rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
3392                         (double)(time_st.enq_sw_max_time * 1000000) /
3393                         rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
3394                         (double)iter,
3395                         (double)(time_st.enq_acc_total_time * 1000000) /
3396                         (double)iter / (double)rte_get_tsc_hz(),
3397                         (double)time_st.enq_acc_min_time,
3398                         (double)(time_st.enq_acc_min_time * 1000000) /
3399                         rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
3400                         (double)(time_st.enq_acc_max_time * 1000000) /
3401                         rte_get_tsc_hz());
3402
3403         printf("Dequeue offload cost latency - one op:\n"
3404                         "\tavg: %lg cycles, %lg us\n"
3405                         "\tmin: %lg cycles, %lg us\n"
3406                         "\tmax: %lg cycles, %lg us\n",
3407                         (double)time_st.deq_total_time / (double)iter,
3408                         (double)(time_st.deq_total_time * 1000000) /
3409                         (double)iter / (double)rte_get_tsc_hz(),
3410                         (double)time_st.deq_min_time,
3411                         (double)(time_st.deq_min_time * 1000000) /
3412                         rte_get_tsc_hz(), (double)time_st.deq_max_time,
3413                         (double)(time_st.deq_max_time * 1000000) /
3414                         rte_get_tsc_hz());
3415
3416         return TEST_SUCCESS;
3417 #endif
3418 }
3419
3420 #ifdef RTE_BBDEV_OFFLOAD_COST
3421 static int
3422 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
3423                 const uint16_t num_to_process, uint16_t burst_sz,
3424                 uint64_t *deq_total_time, uint64_t *deq_min_time,
3425                 uint64_t *deq_max_time)
3426 {
3427         int i, deq_total;
3428         struct rte_bbdev_dec_op *ops[MAX_BURST];
3429         uint64_t deq_start_time, deq_last_time;
3430
3431         /* Test deq offload latency from an empty queue */
3432
3433         for (i = 0, deq_total = 0; deq_total < num_to_process;
3434                         ++i, deq_total += burst_sz) {
3435                 deq_start_time = rte_rdtsc_precise();
3436
3437                 if (unlikely(num_to_process - deq_total < burst_sz))
3438                         burst_sz = num_to_process - deq_total;
3439                 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz);
3440
3441                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3442                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
3443                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
3444                 *deq_total_time += deq_last_time;
3445         }
3446
3447         return i;
3448 }
3449
3450 static int
3451 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
3452                 const uint16_t num_to_process, uint16_t burst_sz,
3453                 uint64_t *deq_total_time, uint64_t *deq_min_time,
3454                 uint64_t *deq_max_time)
3455 {
3456         int i, deq_total;
3457         struct rte_bbdev_enc_op *ops[MAX_BURST];
3458         uint64_t deq_start_time, deq_last_time;
3459
3460         /* Test deq offload latency from an empty queue */
3461         for (i = 0, deq_total = 0; deq_total < num_to_process;
3462                         ++i, deq_total += burst_sz) {
3463                 deq_start_time = rte_rdtsc_precise();
3464
3465                 if (unlikely(num_to_process - deq_total < burst_sz))
3466                         burst_sz = num_to_process - deq_total;
3467                 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz);
3468
3469                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3470                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
3471                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
3472                 *deq_total_time += deq_last_time;
3473         }
3474
3475         return i;
3476 }
3477 #endif
3478
3479 static int
3480 offload_latency_empty_q_test(struct active_device *ad,
3481                 struct test_op_params *op_params)
3482 {
3483 #ifndef RTE_BBDEV_OFFLOAD_COST
3484         RTE_SET_USED(ad);
3485         RTE_SET_USED(op_params);
3486         printf("Offload latency empty dequeue test is disabled.\n");
3487         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
3488         return TEST_SKIPPED;
3489 #else
3490         int iter;
3491         uint64_t deq_total_time, deq_min_time, deq_max_time;
3492         uint16_t burst_sz = op_params->burst_sz;
3493         const uint16_t num_to_process = op_params->num_to_process;
3494         const enum rte_bbdev_op_type op_type = test_vector.op_type;
3495         const uint16_t queue_id = ad->queue_ids[0];
3496         struct rte_bbdev_info info;
3497         const char *op_type_str;
3498
3499         deq_total_time = deq_max_time = 0;
3500         deq_min_time = UINT64_MAX;
3501
3502         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3503                         "BURST_SIZE should be <= %u", MAX_BURST);
3504
3505         rte_bbdev_info_get(ad->dev_id, &info);
3506
3507         op_type_str = rte_bbdev_op_type_str(op_type);
3508         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
3509
3510         printf("+ ------------------------------------------------------- +\n");
3511         printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
3512                         info.dev_name, burst_sz, num_to_process, op_type_str);
3513
3514         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
3515                 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
3516                                 num_to_process, burst_sz, &deq_total_time,
3517                                 &deq_min_time, &deq_max_time);
3518         else
3519                 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
3520                                 num_to_process, burst_sz, &deq_total_time,
3521                                 &deq_min_time, &deq_max_time);
3522
3523         if (iter <= 0)
3524                 return TEST_FAILED;
3525
3526         printf("Empty dequeue offload:\n"
3527                         "\tavg: %lg cycles, %lg us\n"
3528                         "\tmin: %lg cycles, %lg us\n"
3529                         "\tmax: %lg cycles, %lg us\n",
3530                         (double)deq_total_time / (double)iter,
3531                         (double)(deq_total_time * 1000000) / (double)iter /
3532                         (double)rte_get_tsc_hz(), (double)deq_min_time,
3533                         (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
3534                         (double)deq_max_time, (double)(deq_max_time * 1000000) /
3535                         rte_get_tsc_hz());
3536
3537         return TEST_SUCCESS;
3538 #endif
3539 }
3540
3541 static int
3542 throughput_tc(void)
3543 {
3544         return run_test_case(throughput_test);
3545 }
3546
3547 static int
3548 offload_cost_tc(void)
3549 {
3550         return run_test_case(offload_cost_test);
3551 }
3552
3553 static int
3554 offload_latency_empty_q_tc(void)
3555 {
3556         return run_test_case(offload_latency_empty_q_test);
3557 }
3558
3559 static int
3560 latency_tc(void)
3561 {
3562         return run_test_case(latency_test);
3563 }
3564
3565 static int
3566 interrupt_tc(void)
3567 {
3568         return run_test_case(throughput_test);
3569 }
3570
3571 static struct unit_test_suite bbdev_throughput_testsuite = {
3572         .suite_name = "BBdev Throughput Tests",
3573         .setup = testsuite_setup,
3574         .teardown = testsuite_teardown,
3575         .unit_test_cases = {
3576                 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
3577                 TEST_CASES_END() /**< NULL terminate unit test array */
3578         }
3579 };
3580
3581 static struct unit_test_suite bbdev_validation_testsuite = {
3582         .suite_name = "BBdev Validation Tests",
3583         .setup = testsuite_setup,
3584         .teardown = testsuite_teardown,
3585         .unit_test_cases = {
3586                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
3587                 TEST_CASES_END() /**< NULL terminate unit test array */
3588         }
3589 };
3590
3591 static struct unit_test_suite bbdev_latency_testsuite = {
3592         .suite_name = "BBdev Latency Tests",
3593         .setup = testsuite_setup,
3594         .teardown = testsuite_teardown,
3595         .unit_test_cases = {
3596                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
3597                 TEST_CASES_END() /**< NULL terminate unit test array */
3598         }
3599 };
3600
3601 static struct unit_test_suite bbdev_offload_cost_testsuite = {
3602         .suite_name = "BBdev Offload Cost Tests",
3603         .setup = testsuite_setup,
3604         .teardown = testsuite_teardown,
3605         .unit_test_cases = {
3606                 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
3607                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
3608                 TEST_CASES_END() /**< NULL terminate unit test array */
3609         }
3610 };
3611
3612 static struct unit_test_suite bbdev_interrupt_testsuite = {
3613         .suite_name = "BBdev Interrupt Tests",
3614         .setup = interrupt_testsuite_setup,
3615         .teardown = testsuite_teardown,
3616         .unit_test_cases = {
3617                 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
3618                 TEST_CASES_END() /**< NULL terminate unit test array */
3619         }
3620 };
3621
3622 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
3623 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
3624 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
3625 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
3626 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);